{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.383208605Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45548"} {"msg":"region eb5aebaf-3a73-404e-b530-1edd4b09eb59 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.449853375Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.459402969Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.45942631Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions","method":"POST","req_id":"11b1a642-699a-47b0-bf96-1e8af7f38a2d","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":35124,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_06449f3a-80b8-4593-a33c-24c33cea05d7/crucible/regions/eb5aebaf-3a73-404e-b530-1edd4b09eb59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.473913645Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_06449f3a-80b8-4593-a33c-24c33cea05d7/crucible/regions/eb5aebaf-3a73-404e-b530-1edd4b09eb59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.47392812Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"eb5aebaf-3a73-404e-b530-1edd4b09eb59\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eb5aebaf-3a73-404e-b530-1edd4b09eb59\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.567307365Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"eb5aebaf-3a73-404e-b530-1edd4b09eb59","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eb5aebaf-3a73-404e-b530-1edd4b09eb59\" \"--data\" \"/data/regions/eb5aebaf-3a73-404e-b530-1edd4b09eb59\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.573001427Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"eb5aebaf-3a73-404e-b530-1edd4b09eb59","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:52.751041727Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions","method":"POST","req_id":"fe882ae2-57fc-4caf-94b1-b47d0fac7b1b","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":50,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.081325863Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"eb5aebaf-3a73-404e-b530-1edd4b09eb59","component":"worker"} {"msg":"region eb5aebaf-3a73-404e-b530-1edd4b09eb59 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.081370034Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.081516047Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-eb5aebaf-3a73-404e-b530-1edd4b09eb59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.082041208Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eb5aebaf-3a73-404e-b530-1edd4b09eb59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.107143577Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.155817291Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eb5aebaf-3a73-404e-b530-1edd4b09eb59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.170851734Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb5aebaf-3a73-404e-b530-1edd4b09eb59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.171556459Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.17162123Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.171664771Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.171700732Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.18746912Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.195714987Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:53.417868944Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions","method":"POST","req_id":"74c8717d-d546-4cdc-bf90-966c3e664b14","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"region 107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.651826505Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.652032129Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.652049629Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions","method":"POST","req_id":"e3b86afa-9ecb-40d1-a47a-d44d8b44f288","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":284,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_06449f3a-80b8-4593-a33c-24c33cea05d7/crucible/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.658833005Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_06449f3a-80b8-4593-a33c-24c33cea05d7/crucible/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.658847605Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.732925074Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99\" \"--data\" \"/data/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.732955245Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.884588528Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","component":"worker"} {"msg":"region 107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.884638299Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.88471557Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.885056238Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.889568474Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.90800113Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.915330987Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.915623453Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.915663964Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.915704405Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.915738536Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.924671458Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:53:59.952913684Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:00.015345113Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions","method":"POST","req_id":"b8cd4372-1362-46cf-832c-16da09077ec2","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.027619927Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","method":"GET","req_id":"bc8a020c-32be-43e8-b8e7-50f362b7a426","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":77,"response_code":200} {"msg":"path is \"/data/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.030872166Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"dataset is oxp_06449f3a-80b8-4593-a33c-24c33cea05d7/crucible/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.043433105Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"region 107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.05109374Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.051895007Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.051906267Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","method":"DELETE","req_id":"2574fcaa-16c7-4623-b639-f2c319aaf26c","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21063,"response_code":204} {"msg":"disabling downstairs instance: downstairs-107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.052545791Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.055396762Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","method":"GET","req_id":"dc02e7d2-4f30-475f-97a6-eceac36b7d7a","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.067319838Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_06449f3a-80b8-4593-a33c-24c33cea05d7/crucible/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.075863021Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"region":"107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","component":"worker"} {"msg":"region 107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.158058144Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:54:27.178139165Z","hostname":"oxz_crucible_b03b85f0-464d-4b4a-a1d2-e04f3f63e9d4","pid":24541,"uri":"/crucible/0/regions/107bd729-7a7d-4ffc-b8b4-a80dbf0b4e99","method":"GET","req_id":"b5f9ebc5-c838-49e6-b244-81dc5c50ac1c","remote_addr":"[fd00:1122:3344:101::c]:45548","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":50,"response_code":200}