[ Feb 26 17:10:06 Enabled. ]
[ Feb 26 17:10:06 Rereading configuration. ]
[ Feb 26 17:10:09 Rereading configuration. ]
[ Feb 26 17:10:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fd8c702e-57f8-4bbd-a751-a93437545dc2/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:10:13.559496959Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386}
{"msg":"dataset: \"oxp_fd8c702e-57f8-4bbd-a751-a93437545dc2/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:10:13.56026223Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:10:13.560276731Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:10:13.560284331Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:10:13.575816361Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:10:13.663770207Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:07.880502626Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45965"}
{"msg":"region 4220f25b-050b-4e5d-b9ea-80f312d2f9f2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:07.925050559Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:07.927292181Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:07.927322392Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions","method":"POST","req_id":"09ebcf17-b2a6-462b-a992-1987bcc71a83","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18340,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_fd8c702e-57f8-4bbd-a751-a93437545dc2/crucible/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:07.935057196Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fd8c702e-57f8-4bbd-a751-a93437545dc2/crucible/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:07.935075377Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4220f25b-050b-4e5d-b9ea-80f312d2f9f2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.030012296Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"region":"4220f25b-050b-4e5d-b9ea-80f312d2f9f2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4220f25b-050b-4e5d-b9ea-80f312d2f9f2\" \"--data\" \"/data/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.032428463Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"region":"4220f25b-050b-4e5d-b9ea-80f312d2f9f2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.144769624Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions","method":"POST","req_id":"e8f40e61-1637-4df1-b7a4-3fc85356711d","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":67,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.3583527Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"region":"4220f25b-050b-4e5d-b9ea-80f312d2f9f2","component":"worker"}
{"msg":"region 4220f25b-050b-4e5d-b9ea-80f312d2f9f2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.358400181Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.358471063Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.358815052Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.369367085Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.409529327Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.412901561Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.413736613Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.413816266Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.413868377Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.413913909Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.418036463Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.427200457Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:08.669519688Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions","method":"POST","req_id":"01f3dbeb-b22e-4d12-8c9b-c22457fec006","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.295316839Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","method":"GET","req_id":"c1a0cfa5-d0cd-4d09-9597-6b0418819db9","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":75,"response_code":200}
{"msg":"path is \"/data/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.298162778Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"dataset is oxp_fd8c702e-57f8-4bbd-a751-a93437545dc2/crucible/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.305861241Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"region 4220f25b-050b-4e5d-b9ea-80f312d2f9f2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.312847954Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.312981568Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.312997149Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","method":"DELETE","req_id":"ccbee5b3-8a84-45e8-984f-b02e0754cf8e","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":14888,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-4220f25b-050b-4e5d-b9ea-80f312d2f9f2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.313462171Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.315754635Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","method":"GET","req_id":"b5d01e4b-b38a-433d-b9a6-c2e28a62dd15","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":30,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.318749888Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fd8c702e-57f8-4bbd-a751-a93437545dc2/crucible/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.325270509Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"region":"4220f25b-050b-4e5d-b9ea-80f312d2f9f2","component":"worker"}
{"msg":"region 4220f25b-050b-4e5d-b9ea-80f312d2f9f2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.430736509Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-26T17:11:09.455108744Z","hostname":"oxz_crucible_badfcfc3-d284-4306-8c73-4709366a64f5","pid":23386,"uri":"/crucible/0/regions/4220f25b-050b-4e5d-b9ea-80f312d2f9f2","method":"GET","req_id":"f7a2082b-a251-4765-a8bb-127294b85913","remote_addr":"[fd00:1122:3344:101::b]:45965","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":47,"response_code":200}