[ Mar 24 20:56:46 Enabled. ]
[ Mar 24 20:56:47 Rereading configuration. ]
[ Mar 24 20:56:49 Rereading configuration. ]
[ Mar 24 20:56:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3f60d358-14b7-4507-95d6-0257473262bc/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:56:55.069020585Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092}
{"msg":"dataset: \"oxp_3f60d358-14b7-4507-95d6-0257473262bc/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:56:55.069874318Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:56:55.069884438Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:56:55.069889668Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:56:55.083899014Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:56:55.223083281Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.73299897Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45728"}
{"msg":"region 3f51a990-6e6b-4de1-9eed-bffcb03cd053 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.758985661Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.761870109Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.76191315Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions","method":"POST","req_id":"57e1c9fd-03b3-4d70-9021-1eec03359b8b","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":14386,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_3f60d358-14b7-4507-95d6-0257473262bc/crucible/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.770268795Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_3f60d358-14b7-4507-95d6-0257473262bc/crucible/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.770283675Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3f51a990-6e6b-4de1-9eed-bffcb03cd053\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.834281321Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"region":"3f51a990-6e6b-4de1-9eed-bffcb03cd053","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3f51a990-6e6b-4de1-9eed-bffcb03cd053\" \"--data\" \"/data/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:20.836293175Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"region":"3f51a990-6e6b-4de1-9eed-bffcb03cd053","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.071478497Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions","method":"POST","req_id":"690a8445-e1dc-4750-b9e5-69fbfe1a1c35","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.189330535Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"region":"3f51a990-6e6b-4de1-9eed-bffcb03cd053","component":"worker"}
{"msg":"region 3f51a990-6e6b-4de1-9eed-bffcb03cd053 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.189394047Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.189608473Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.189982853Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.197397503Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.220347802Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.22364363Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.224173085Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.224230786Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.224267867Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.224300768Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.229337014Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.234127703Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:21.387421556Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions","method":"POST","req_id":"e85dd157-9860-4f40-a631-68b2b5a4ebad","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.092147279Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","method":"GET","req_id":"d311a4db-d515-412e-9f3c-09d9655af91b","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"path is \"/data/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.094908114Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"dataset is oxp_3f60d358-14b7-4507-95d6-0257473262bc/crucible/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.108823299Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"region 3f51a990-6e6b-4de1-9eed-bffcb03cd053 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.115854478Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.116057974Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","method":"DELETE","req_id":"b86db46e-3f54-48ac-9288-182d9cc38f37","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21169,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.116102455Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-3f51a990-6e6b-4de1-9eed-bffcb03cd053 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.117082081Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.11960498Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","method":"GET","req_id":"45372d9f-5345-4d4e-acca-caae39a69eca","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":30,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.121890011Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3f60d358-14b7-4507-95d6-0257473262bc/crucible/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.129543977Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"region":"3f51a990-6e6b-4de1-9eed-bffcb03cd053","component":"worker"}
{"msg":"region 3f51a990-6e6b-4de1-9eed-bffcb03cd053 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.198838546Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T20:57:22.23125832Z","hostname":"oxz_crucible_26a48484-1e5d-480d-beb5-c51c0709a460","pid":24092,"uri":"/crucible/0/regions/3f51a990-6e6b-4de1-9eed-bffcb03cd053","method":"GET","req_id":"40263e9a-5b3e-4589-a5da-dc8e429e91f7","remote_addr":"[fd00:1122:3344:101::c]:45728","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":46,"response_code":200}