[ Feb 3 21:14:55 Enabled. ] [ Feb 3 21:14:55 Rereading configuration. ] [ Feb 3 21:14:59 Rereading configuration. ] [ Feb 3 21:15:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:05.052812344Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130} {"msg":"dataset: \"oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:05.053657679Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130} {"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:05.053669029Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:05.05367597Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:05.072763174Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:05.423698561Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.627402112Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:48437"} {"msg":"region 187bd333-eaec-4cca-a577-42269099ef70 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.671365495Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.674435457Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.674477128Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"c59bd8b6-28b1-46d6-a78b-db40b9bfc9ca","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20087,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.682735057Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.682754657Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"187bd333-eaec-4cca-a577-42269099ef70\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/187bd333-eaec-4cca-a577-42269099ef70\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.76494861Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"187bd333-eaec-4cca-a577-42269099ef70","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"187bd333-eaec-4cca-a577-42269099ef70\" \"--data\" \"/data/regions/187bd333-eaec-4cca-a577-42269099ef70\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.767315051Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"187bd333-eaec-4cca-a577-42269099ef70","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:37.871951909Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"b256a9fd-0fb8-467d-ab86-93eaf917599f","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.117867577Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"187bd333-eaec-4cca-a577-42269099ef70","component":"worker"} {"msg":"region 187bd333-eaec-4cca-a577-42269099ef70 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.117906218Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.118125855Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.118468545Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.126010392Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.140953292Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.144394755Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.145095176Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.145149748Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.145190359Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.14522477Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.150437417Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.155046896Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.382674753Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"9aaacab0-4f1c-4917-bb8f-f626a169e75d","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.939612738Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/187bd333-eaec-4cca-a577-42269099ef70","method":"GET","req_id":"857a4061-57fc-467b-99bd-699309b2b37e","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":76,"response_code":200} {"msg":"path is \"/data/regions/187bd333-eaec-4cca-a577-42269099ef70\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.942712162Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"dataset is oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/187bd333-eaec-4cca-a577-42269099ef70","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.952214437Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"region 187bd333-eaec-4cca-a577-42269099ef70 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.960537448Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.960757694Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/187bd333-eaec-4cca-a577-42269099ef70","method":"DELETE","req_id":"7f9952c4-e29c-4d92-afbb-b18df0749eb5","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18064,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.960810556Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-187bd333-eaec-4cca-a577-42269099ef70 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.961763755Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.964816337Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/187bd333-eaec-4cca-a577-42269099ef70","method":"GET","req_id":"c435dc29-84ed-4304-8885-2b7c8eb17445","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":38,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.96724516Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/187bd333-eaec-4cca-a577-42269099ef70\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:38.976306532Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"187bd333-eaec-4cca-a577-42269099ef70","component":"worker"} {"msg":"region 187bd333-eaec-4cca-a577-42269099ef70 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:39.055432843Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:15:39.064851386Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/187bd333-eaec-4cca-a577-42269099ef70","method":"GET","req_id":"e1c79a81-ce5f-4a85-8500-ee17e99542da","remote_addr":"[fd00:1122:3344:101::c]:48437","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.68130097Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:38999"} {"msg":"region 1c943ece-4bfc-405f-888b-a39d192a9dfe state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.685229578Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.685334911Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"48a44c85-4dab-4d48-8994-ea8ab4a81339","remote_addr":"[fd00:1122:3344:101::c]:38999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":3964,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.685348822Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/1c943ece-4bfc-405f-888b-a39d192a9dfe","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.693297121Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/1c943ece-4bfc-405f-888b-a39d192a9dfe","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.693321251Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"1c943ece-4bfc-405f-888b-a39d192a9dfe\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1c943ece-4bfc-405f-888b-a39d192a9dfe\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.793202826Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"1c943ece-4bfc-405f-888b-a39d192a9dfe","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1c943ece-4bfc-405f-888b-a39d192a9dfe\" \"--data\" \"/data/regions/1c943ece-4bfc-405f-888b-a39d192a9dfe\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.793240407Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"1c943ece-4bfc-405f-888b-a39d192a9dfe","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.954127988Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"22208993-955d-42b3-9c1f-70963d00639a","remote_addr":"[fd00:1122:3344:101::c]:38999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.981605444Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"1c943ece-4bfc-405f-888b-a39d192a9dfe","component":"worker"} {"msg":"region 1c943ece-4bfc-405f-888b-a39d192a9dfe state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.981626785Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.981755029Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-1c943ece-4bfc-405f-888b-a39d192a9dfe","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:02.982561103Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1c943ece-4bfc-405f-888b-a39d192a9dfe","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.014148063Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.049692693Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1c943ece-4bfc-405f-888b-a39d192a9dfe","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.0562521Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1c943ece-4bfc-405f-888b-a39d192a9dfe","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.0565855Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.056640582Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.056694083Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.056739215Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.066692894Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.090283814Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:03.564120729Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"a4f9657d-835f-4cdf-b3ee-e1809efe9f26","remote_addr":"[fd00:1122:3344:101::c]:38999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200} {"msg":"region 6e7e219c-77ff-4ab7-bfa6-30155554a603 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.465620808Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.465795953Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"49ae3b10-7a32-4f7d-aba6-471f3eef13f8","remote_addr":"[fd00:1122:3344:101::c]:38999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":231,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.465826444Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.472733662Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.472753332Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6e7e219c-77ff-4ab7-bfa6-30155554a603\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.556575974Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"6e7e219c-77ff-4ab7-bfa6-30155554a603","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6e7e219c-77ff-4ab7-bfa6-30155554a603\" \"--data\" \"/data/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.556625506Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"6e7e219c-77ff-4ab7-bfa6-30155554a603","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.707498625Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"8e9f1c13-edde-426e-80cf-91b90f927663","remote_addr":"[fd00:1122:3344:101::c]:38999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.73857795Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"6e7e219c-77ff-4ab7-bfa6-30155554a603","component":"worker"} {"msg":"region 6e7e219c-77ff-4ab7-bfa6-30155554a603 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.738606171Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.738719384Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.743607851Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.76551502Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.793596595Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.802578065Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.802871994Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.802938626Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.802980647Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.803015858Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.814410771Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:11.826368421Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:12.208042602Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions","method":"POST","req_id":"d9b38634-e429-4c90-b396-29d14c8a0085","remote_addr":"[fd00:1122:3344:101::c]:38999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.127157807Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43524"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.127322132Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","method":"GET","req_id":"e8f3e82f-3ce6-46bb-8227-4ee4b364c445","remote_addr":"[fd00:1122:3344:101::c]:43524","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"path is \"/data/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.131702714Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"dataset is oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.141286972Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"region 6e7e219c-77ff-4ab7-bfa6-30155554a603 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.148122077Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.148269592Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","method":"DELETE","req_id":"95d68cf5-e7ac-4088-8339-76769e851f65","remote_addr":"[fd00:1122:3344:101::c]:43524","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":16578,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.148290833Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-6e7e219c-77ff-4ab7-bfa6-30155554a603 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.149430587Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.1511953Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","method":"GET","req_id":"3b416823-2ab2-49bb-84c1-67774a4fb8f5","remote_addr":"[fd00:1122:3344:101::c]:43524","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":39,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.162478369Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_437ab7a9-1181-4e21-98e5-7e695d8464b5/crucible/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.170723268Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"region":"6e7e219c-77ff-4ab7-bfa6-30155554a603","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.213773582Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","method":"GET","req_id":"72067125-66f0-4f83-8d1e-eec422c1429a","remote_addr":"[fd00:1122:3344:101::c]:43524","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":69,"response_code":200} {"msg":"region 6e7e219c-77ff-4ab7-bfa6-30155554a603 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.226770422Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-03T21:27:44.391928821Z","hostname":"oxz_crucible_b39bc9e1-64d7-4a37-9fea-5608a50ea04f","pid":130,"uri":"/crucible/0/regions/6e7e219c-77ff-4ab7-bfa6-30155554a603","method":"GET","req_id":"96f93656-4bba-4ba8-acfb-2daf25d24ca6","remote_addr":"[fd00:1122:3344:101::c]:43524","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":42,"response_code":200}