[ Oct 31 05:29:58 Enabled. ]
[ Oct 31 05:29:58 Rereading configuration. ]
[ Oct 31 05:29:59 Rereading configuration. ]
[ Oct 31 05:30:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7cf21e44-3124-421b-bd12-fb2800fa9e14/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: f85cf0dc3631e2f0c13384787efb0efc1b642162\nCommit timestamp: 2026-10-10T00:38:51.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:30:03.483927174Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151}
{"msg":"dataset: \"oxp_7cf21e44-3124-421b-bd12-fb2800fa9e14/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:30:03.484197631Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:30:03.484220051Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:30:03.484230281Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:30:03.518854145Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:30:03.634550676Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.087433906Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:64228"}
{"msg":"region c0f8cdf8-d1fc-45a8-8414-4a631252a1a7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.132592357Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.140647973Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.145242615Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions","method":"POST","req_id":"8284eb33-05e7-40b2-b5f1-7225e63d760d","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22607,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_7cf21e44-3124-421b-bd12-fb2800fa9e14/crucible/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.148759751Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7cf21e44-3124-421b-bd12-fb2800fa9e14/crucible/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.148784632Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c0f8cdf8-d1fc-45a8-8414-4a631252a1a7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.192767954Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"region":"c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c0f8cdf8-d1fc-45a8-8414-4a631252a1a7\" \"--data\" \"/data/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.192805555Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"region":"c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.456651757Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions","method":"POST","req_id":"3691348a-8705-409f-bc82-e5e1e4e58726","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":69,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.531678296Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"region":"c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","component":"worker"}
{"msg":"region c0f8cdf8-d1fc-45a8-8414-4a631252a1a7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.533654404Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.533805467Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.534189787Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.544296993Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.55730824Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.558549241Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.559281359Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.55933924Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.559378031Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.559412562Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.561608085Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.56384742Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:41:41.842147904Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions","method":"POST","req_id":"43f6ec66-1898-478f-ab1e-8d109a2f955b","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.04818876Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","method":"GET","req_id":"c911971d-28ba-4583-b49c-92dea49c1ec0","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"path is \"/data/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.052418923Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"dataset is oxp_7cf21e44-3124-421b-bd12-fb2800fa9e14/crucible/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.062597321Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"region c0f8cdf8-d1fc-45a8-8414-4a631252a1a7 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.068746341Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.068903465Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.068917495Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","method":"DELETE","req_id":"897fb407-a5ed-4a1d-b994-70c9b3177c29","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16528,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-c0f8cdf8-d1fc-45a8-8414-4a631252a1a7 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.069801517Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.072195505Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","method":"GET","req_id":"aa98d44d-4d59-4c50-9260-716e735c4e71","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":46,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.072433871Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7cf21e44-3124-421b-bd12-fb2800fa9e14/crucible/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.079980085Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"region":"c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","component":"worker"}
{"msg":"region c0f8cdf8-d1fc-45a8-8414-4a631252a1a7 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.122260876Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-31T05:42:07.203963607Z","hostname":"oxz_crucible_6694e0a7-e6f3-437d-9cb6-09009097b33d","pid":27151,"uri":"/crucible/0/regions/c0f8cdf8-d1fc-45a8-8414-4a631252a1a7","method":"GET","req_id":"a3ba5fab-2fa6-41cc-a4d6-66c6f967870c","remote_addr":"[fd00:1122:3344:101::c]:64228","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":54,"response_code":200}