[ Jan 25 03:52:59 Enabled. ]
[ Jan 25 03:52:59 Rereading configuration. ]
[ Jan 25 03:53:04 Rereading configuration. ]
[ Jan 25 03:53:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:53:09.383098479Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:53:09.384322559Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:53:09.384333729Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:53:09.400486031Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:53:09.484082529Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.369978624Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37054"}
{"msg":"region 406bd748-9f4a-4051-859b-76d8ad2bc95e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.393474724Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.395893893Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions","method":"POST","req_id":"25cf3fb2-22b3-4113-b74b-2acb2c9941ec","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":10849,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.398150858Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.406863649Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.40689837Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"406bd748-9f4a-4051-859b-76d8ad2bc95e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.460269695Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"406bd748-9f4a-4051-859b-76d8ad2bc95e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"406bd748-9f4a-4051-859b-76d8ad2bc95e\" \"--data\" \"/data/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.462251793Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"406bd748-9f4a-4051-859b-76d8ad2bc95e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.583634978Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"406bd748-9f4a-4051-859b-76d8ad2bc95e","component":"worker"}
{"msg":"region 406bd748-9f4a-4051-859b-76d8ad2bc95e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.583694939Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.583799572Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.584300684Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.588426824Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.604228957Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.619470397Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.620028341Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.62204281Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.622116501Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.622179363Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.626852866Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.632308959Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:06.726930274Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions","method":"POST","req_id":"6ecb3277-b6ed-4131-be83-e4ff0a0aa854","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.40115834Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","method":"GET","req_id":"4e50a5e0-cb2d-4e9a-be19-a29b825d656d","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"path is \"/data/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.404770507Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"dataset is oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.416457111Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"region 406bd748-9f4a-4051-859b-76d8ad2bc95e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.427869278Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.428015782Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-406bd748-9f4a-4051-859b-76d8ad2bc95e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.429107428Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.430038191Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","method":"DELETE","req_id":"2521b795-8255-401d-abf6-32064a1e97f9","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25270,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.433277109Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","method":"GET","req_id":"92f0d9d7-e6be-4423-9801-c242ff451330","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.433819872Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.447260418Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"406bd748-9f4a-4051-859b-76d8ad2bc95e","component":"worker"}
{"msg":"region 406bd748-9f4a-4051-859b-76d8ad2bc95e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.519599193Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:07.583913093Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions/406bd748-9f4a-4051-859b-76d8ad2bc95e","method":"GET","req_id":"7a96b684-f627-4c7b-8c1b-82265f486dd1","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region 8d746af8-4b81-470e-9b45-bd136227e996 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.686484061Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.686623795Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions","method":"POST","req_id":"9bb3725b-fe70-4420-9eaf-aa3acdaa4c6f","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":206,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.686640275Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible/regions/8d746af8-4b81-470e-9b45-bd136227e996","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.694828694Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a23e7ed6-7d83-458b-8c71-7390568d1c9a/crucible/regions/8d746af8-4b81-470e-9b45-bd136227e996","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.694858544Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8d746af8-4b81-470e-9b45-bd136227e996\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8d746af8-4b81-470e-9b45-bd136227e996\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.74497284Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"8d746af8-4b81-470e-9b45-bd136227e996","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8d746af8-4b81-470e-9b45-bd136227e996\" \"--data\" \"/data/regions/8d746af8-4b81-470e-9b45-bd136227e996\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.746948258Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"8d746af8-4b81-470e-9b45-bd136227e996","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.807345083Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"region":"8d746af8-4b81-470e-9b45-bd136227e996","component":"worker"}
{"msg":"region 8d746af8-4b81-470e-9b45-bd136227e996 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.807389904Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.807475046Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8d746af8-4b81-470e-9b45-bd136227e996","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.809023804Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8d746af8-4b81-470e-9b45-bd136227e996","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.812045767Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.818840912Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8d746af8-4b81-470e-9b45-bd136227e996","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.821470406Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8d746af8-4b81-470e-9b45-bd136227e996","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.821799754Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.821867425Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.821923447Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.821990879Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.825217767Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.829380668Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-25T03:54:09.836307406Z","hostname":"oxz_crucible_5215aca1-6983-4917-89ca-29d771b18215","pid":22484,"uri":"/crucible/0/regions","method":"POST","req_id":"518f4cee-9ddb-4111-a8ad-771b9eb50cdb","remote_addr":"[fd00:1122:3344:101::a]:37054","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}