[ May  4 19:29:27 Enabled. ]
[ May  4 19:29:27 Rereading configuration. ]
[ May  4 19:29:29 Rereading configuration. ]
[ May  4 19:29:32 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ff3362e5-38a8-446f-bf2a-41356d86e11b/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:32.334564127Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439}
{"msg":"dataset: \"oxp_ff3362e5-38a8-446f-bf2a-41356d86e11b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:32.335299002Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:32.335308752Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:32.335313972Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:32.347944206Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:32.427240231Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.106609172Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:36656"}
{"msg":"region 156c04e9-2a87-49c0-8abf-7f23cdcd969e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.131779538Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.135233678Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.135275809Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"71a0e2a3-9428-452f-a03c-6d65eb8497f2","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":14968,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_ff3362e5-38a8-446f-bf2a-41356d86e11b/crucible/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.145007944Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ff3362e5-38a8-446f-bf2a-41356d86e11b/crucible/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.145023585Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"156c04e9-2a87-49c0-8abf-7f23cdcd969e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.237091256Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"region":"156c04e9-2a87-49c0-8abf-7f23cdcd969e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"156c04e9-2a87-49c0-8abf-7f23cdcd969e\" \"--data\" \"/data/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.239624807Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"region":"156c04e9-2a87-49c0-8abf-7f23cdcd969e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.449128511Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"948e8e30-5210-4ba5-9930-1ce7a3a06320","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.661681066Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"region":"156c04e9-2a87-49c0-8abf-7f23cdcd969e","component":"worker"}
{"msg":"region 156c04e9-2a87-49c0-8abf-7f23cdcd969e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.661756258Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.662034183Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.66236602Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.685971665Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.736095083Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.749923841Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.750670996Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.750743337Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.750789018Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.750824529Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.759608366Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:58.766534185Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.102962661Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"2d002a84-51b4-4e1c-b1fc-89dcccbd1561","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.754540736Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","method":"GET","req_id":"810d2cce-bea1-47b3-88f4-4a61bb48e168","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":79,"response_code":200}
{"msg":"path is \"/data/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.757267551Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"dataset is oxp_ff3362e5-38a8-446f-bf2a-41356d86e11b/crucible/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.76568785Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"region 156c04e9-2a87-49c0-8abf-7f23cdcd969e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.774393976Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.774564669Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.77460926Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","method":"DELETE","req_id":"555ba20e-b9c6-4a7b-b36f-5f49f471511c","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":17316,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-156c04e9-2a87-49c0-8abf-7f23cdcd969e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.775436146Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.778716852Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","method":"GET","req_id":"c2418058-916c-4825-b892-ba3c68562a0c","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.7890455Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ff3362e5-38a8-446f-bf2a-41356d86e11b/crucible/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.798406388Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"region":"156c04e9-2a87-49c0-8abf-7f23cdcd969e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.836517535Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","method":"GET","req_id":"b21c4c91-5736-4d35-a097-395ad07b6a1a","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"region 156c04e9-2a87-49c0-8abf-7f23cdcd969e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.922684307Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-04T19:29:59.980627692Z","hostname":"oxz_crucible_74f9f67d-67b5-4f3b-9579-79f1c6ab666c","pid":24439,"uri":"/crucible/0/regions/156c04e9-2a87-49c0-8abf-7f23cdcd969e","method":"GET","req_id":"a112b9d7-8c9d-4275-b441-a32a50d3d3d5","remote_addr":"[fd00:1122:3344:101::c]:36656","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":39,"response_code":200}