[ Oct 27 05:27:36 Enabled. ]
[ Oct 27 05:27:36 Rereading configuration. ]
[ Oct 27 05:27:38 Rereading configuration. ]
[ Oct 27 05:27:41 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:27:41.664109455Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821}
{"msg":"dataset: \"oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:27:41.664322149Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:27:41.66433558Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:27:41.66434341Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:27:41.677908806Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:27:41.746060462Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.010817801Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37102"}
{"msg":"region d29dd07a-6b3f-4300-97f8-250c8c616b14 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.040390397Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.044643259Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.048574905Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions","method":"POST","req_id":"481baa05-deb5-46b0-8b8a-4583c8373305","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14846,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.051281274Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.051310875Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d29dd07a-6b3f-4300-97f8-250c8c616b14\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.091309007Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"d29dd07a-6b3f-4300-97f8-250c8c616b14","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d29dd07a-6b3f-4300-97f8-250c8c616b14\" \"--data\" \"/data/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.093223369Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"d29dd07a-6b3f-4300-97f8-250c8c616b14","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.206204523Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"d29dd07a-6b3f-4300-97f8-250c8c616b14","component":"worker"}
{"msg":"region d29dd07a-6b3f-4300-97f8-250c8c616b14 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.209986176Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.210132679Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.212392078Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.216827555Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.225155447Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.226328902Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.227076788Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.22714145Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.227177591Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.227199831Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.229266536Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.231386242Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.306038441Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions","method":"POST","req_id":"187303dc-44f2-4e8e-9a2c-780b52ed2fb6","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.829995308Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","method":"GET","req_id":"b2342acc-30f7-4de7-b2c0-71c94ad40073","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":179,"response_code":200}
{"msg":"path is \"/data/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.832533123Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"dataset is oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.839776532Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"region d29dd07a-6b3f-4300-97f8-250c8c616b14 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.847216624Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.847359157Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.847370927Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","method":"DELETE","req_id":"9358f6f7-f6d1-4b85-aaf7-e9f882d950fd","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14868,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-d29dd07a-6b3f-4300-97f8-250c8c616b14 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.848534972Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.850343962Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","method":"GET","req_id":"2025200d-3182-4ac8-89a7-55fb29f17af9","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.850922045Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.858565661Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"d29dd07a-6b3f-4300-97f8-250c8c616b14","component":"worker"}
{"msg":"region d29dd07a-6b3f-4300-97f8-250c8c616b14 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.910843382Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:02.944941865Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions/d29dd07a-6b3f-4300-97f8-250c8c616b14","method":"GET","req_id":"8bee7ccc-b7c5-4966-9ba2-a754617364f5","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region 9877bc44-0d60-4f0e-bf56-cf08f89ea140 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:04.984749166Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:04.984886Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions","method":"POST","req_id":"71e5d043-da6d-4e9b-aa52-801f2e235a11","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":183,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:04.98492895Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible/regions/9877bc44-0d60-4f0e-bf56-cf08f89ea140","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:04.989614252Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0eaa780a-dda0-4dd3-9a51-64a5721d45df/crucible/regions/9877bc44-0d60-4f0e-bf56-cf08f89ea140","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:04.989629663Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9877bc44-0d60-4f0e-bf56-cf08f89ea140\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9877bc44-0d60-4f0e-bf56-cf08f89ea140\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.033945959Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"9877bc44-0d60-4f0e-bf56-cf08f89ea140","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9877bc44-0d60-4f0e-bf56-cf08f89ea140\" \"--data\" \"/data/regions/9877bc44-0d60-4f0e-bf56-cf08f89ea140\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.03398063Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"9877bc44-0d60-4f0e-bf56-cf08f89ea140","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.075426204Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"region":"9877bc44-0d60-4f0e-bf56-cf08f89ea140","component":"worker"}
{"msg":"region 9877bc44-0d60-4f0e-bf56-cf08f89ea140 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.075463945Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.075577638Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9877bc44-0d60-4f0e-bf56-cf08f89ea140","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.076843015Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9877bc44-0d60-4f0e-bf56-cf08f89ea140","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.078012581Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.081253602Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9877bc44-0d60-4f0e-bf56-cf08f89ea140","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.082437067Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9877bc44-0d60-4f0e-bf56-cf08f89ea140","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.082769464Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.082814065Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.082857366Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.082894857Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.084932722Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.0880924Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-27T05:29:05.4200234Z","hostname":"oxz_crucible_180ad5df-29e7-4b17-893a-561378be870a","pid":22821,"uri":"/crucible/0/regions","method":"POST","req_id":"26a2fb44-c0a6-4a8b-a1c5-19c89957900a","remote_addr":"[fd00:1122:3344:101::c]:37102","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":58,"response_code":200}