[ Dec 22 16:22:11 Enabled. ] [ Dec 22 16:22:11 Rereading configuration. ] [ Dec 22 16:22:14 Rereading configuration. ] [ Dec 22 16:22:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:22:22.25500743Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374} {"msg":"dataset: \"oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:22:22.255721899Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:22:22.255730679Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:22:22.255738939Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:22:22.266497851Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:22:22.361741984Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.195394745Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:64513"} {"msg":"region 87f90351-6268-407b-b9a9-641ba7bedf7b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.222986867Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.226633403Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.229290212Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions","method":"POST","req_id":"42595388-2cd8-4559-a8ce-4fb8564e2338","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17017,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.23568947Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.23570854Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"87f90351-6268-407b-b9a9-641ba7bedf7b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/87f90351-6268-407b-b9a9-641ba7bedf7b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.290446213Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"87f90351-6268-407b-b9a9-641ba7bedf7b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"87f90351-6268-407b-b9a9-641ba7bedf7b\" \"--data\" \"/data/regions/87f90351-6268-407b-b9a9-641ba7bedf7b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.293291998Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"87f90351-6268-407b-b9a9-641ba7bedf7b","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.566681634Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions","method":"POST","req_id":"8a788b13-e605-4852-8747-916cdc4073a7","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.629751855Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"87f90351-6268-407b-b9a9-641ba7bedf7b","component":"worker"} {"msg":"region 87f90351-6268-407b-b9a9-641ba7bedf7b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.629837167Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.630056093Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.630451513Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.635147556Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.649164183Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.65210731Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.652654824Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.652709995Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.652753017Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.652790578Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.656388322Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:35.660101599Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.108023513Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions","method":"POST","req_id":"f42c27d4-3e53-4981-bbeb-5f05a2355d56","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.592991077Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","method":"GET","req_id":"bb34b6b4-84dc-4b8e-ad12-7915ce65636c","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":90,"response_code":200} {"msg":"path is \"/data/regions/87f90351-6268-407b-b9a9-641ba7bedf7b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.595845392Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"dataset is oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.603625976Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"region 87f90351-6268-407b-b9a9-641ba7bedf7b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.610191397Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.610323181Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","method":"DELETE","req_id":"08ea6918-5980-40a5-ad7c-5e4f53e37f56","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":14522,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.610347941Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-87f90351-6268-407b-b9a9-641ba7bedf7b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.61144705Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.612398925Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","method":"GET","req_id":"e53ada23-e008-433f-98a6-805421c3ef6f","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":24,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.615482296Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible/regions/87f90351-6268-407b-b9a9-641ba7bedf7b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.622933381Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"87f90351-6268-407b-b9a9-641ba7bedf7b","component":"worker"} {"msg":"region 87f90351-6268-407b-b9a9-641ba7bedf7b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.693911469Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:36.711810407Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions/87f90351-6268-407b-b9a9-641ba7bedf7b","method":"GET","req_id":"fdf11f85-68d6-403f-ad40-e76bc5d12fb7","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":53,"response_code":200} {"msg":"region 0f719376-164e-4cd7-80c8-f440180c9186 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.259532238Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.259647861Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions","method":"POST","req_id":"828599e8-fc74-435c-bcc0-ab1de3124713","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":173,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.259671092Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible/regions/0f719376-164e-4cd7-80c8-f440180c9186","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.265986697Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_ce061f24-b5c8-4a3d-826a-6d56ed590650/crucible/regions/0f719376-164e-4cd7-80c8-f440180c9186","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.266002018Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"0f719376-164e-4cd7-80c8-f440180c9186\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0f719376-164e-4cd7-80c8-f440180c9186\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.316020096Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"0f719376-164e-4cd7-80c8-f440180c9186","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0f719376-164e-4cd7-80c8-f440180c9186\" \"--data\" \"/data/regions/0f719376-164e-4cd7-80c8-f440180c9186\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.316066458Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"0f719376-164e-4cd7-80c8-f440180c9186","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.383911923Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"region":"0f719376-164e-4cd7-80c8-f440180c9186","component":"worker"} {"msg":"region 0f719376-164e-4cd7-80c8-f440180c9186 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.383948574Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.384037357Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-0f719376-164e-4cd7-80c8-f440180c9186","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.38531849Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0f719376-164e-4cd7-80c8-f440180c9186","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.388022591Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.395933778Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0f719376-164e-4cd7-80c8-f440180c9186","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.398859015Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0f719376-164e-4cd7-80c8-f440180c9186","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.399092761Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.399137342Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.399173093Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.399209384Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.402335746Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.406215347Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-22T16:23:39.628276199Z","hostname":"oxz_crucible_304ade93-ddca-4814-900b-c7cece36cb56","pid":23374,"uri":"/crucible/0/regions","method":"POST","req_id":"7377e2a5-5ade-4a90-af47-8f1bb8fabdfb","remote_addr":"[fd00:1122:3344:101::c]:64513","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":200}