[ Jan 16 03:31:46 Enabled. ]
[ Jan 16 03:31:46 Rereading configuration. ]
[ Jan 16 03:31:48 Rereading configuration. ]
[ Jan 16 03:31:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_06811366-c76f-444f-8d88-46eae8332757/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:57.200518458Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170}
{"msg":"dataset: \"oxp_06811366-c76f-444f-8d88-46eae8332757/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:57.201123637Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:57.201132977Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:57.201138367Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:57.213183778Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:57.409573466Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.216796245Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61076"}
{"msg":"region 40e463cb-af0a-4f90-a80b-9f64aecaf603 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.253522054Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.256097383Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.256138324Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions","method":"POST","req_id":"5829211a-244e-4aa3-91b5-6f1f4b5abed2","remote_addr":"[fd00:1122:3344:101::c]:61076","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19736,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.266408627Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.266424298Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"40e463cb-af0a-4f90-a80b-9f64aecaf603\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.339232657Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"region":"40e463cb-af0a-4f90-a80b-9f64aecaf603","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"40e463cb-af0a-4f90-a80b-9f64aecaf603\" \"--data\" \"/data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.34148095Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"region":"40e463cb-af0a-4f90-a80b-9f64aecaf603","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.486082283Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions","method":"POST","req_id":"cabde80e-0098-44d9-bdf2-f3fa0a8fa426","remote_addr":"[fd00:1122:3344:101::c]:61076","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.71047025Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"region":"40e463cb-af0a-4f90-a80b-9f64aecaf603","component":"worker"}
{"msg":"region 40e463cb-af0a-4f90-a80b-9f64aecaf603 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.710542881Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.710710563Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.711034808Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.721239711Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.736945146Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.739827809Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.740369257Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.740434918Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.740474029Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.740507379Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.745088028Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.749454103Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.810666629Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions","method":"POST","req_id":"7e7011dd-2190-4257-83a2-0e2efb1b56f0","remote_addr":"[fd00:1122:3344:101::c]:61076","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":51,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.865920522Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:41290"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.866519181Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","method":"GET","req_id":"e8b95288-85c1-4fbe-9c29-a93a2424e0a1","remote_addr":"[fd00:1122:3344:101::c]:41290","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"path is \"/data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.869565697Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"dataset is oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.874950627Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"snapshot is oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603@0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.881016238Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\", \"0bc422a3-9d51-48e9-a8f9-22b4a62ea95f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.881041398Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.890388358Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603/snapshots/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","method":"GET","req_id":"c928c9b8-5dc4-4a3b-824a-a9f0d87c8247","remote_addr":"[fd00:1122:3344:101::c]:41290","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20841,"response_code":200}
{"msg":"path is \"/data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.893661647Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"dataset is oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.899530265Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"snapshot is oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603@0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.905851059Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\", \"0bc422a3-9d51-48e9-a8f9-22b4a62ea95f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.905867969Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 40e463cb-af0a-4f90-a80b-9f64aecaf603-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.912802643Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.91328006Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603/snapshots/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f/run","method":"POST","req_id":"e26b3fd5-17f7-43f1-aecb-081036083fd6","remote_addr":"[fd00:1122:3344:101::c]:41290","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19637,"response_code":200}
{"msg":"applying SMF actions for region 40e463cb-af0a-4f90-a80b-9f64aecaf603 running snapshot 0bc422a3-9d51-48e9-a8f9-22b4a62ea95f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.913298861Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-40e463cb-af0a-4f90-a80b-9f64aecaf603-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.934241474Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-40e463cb-af0a-4f90-a80b-9f64aecaf603-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.94536352Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-40e463cb-af0a-4f90-a80b-9f64aecaf603-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.955026965Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603/.zfs/snapshot/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.9553207Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.95536591Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.955406391Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.955442881Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.955488992Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.95872299Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.963273708Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"worker"}
{"msg":"region 40e463cb-af0a-4f90-a80b-9f64aecaf603 running snapshot 0bc422a3-9d51-48e9-a8f9-22b4a62ea95f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.963283619Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"path is \"/data/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:08.264415964Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"dataset is oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:08.270685198Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"datafile"}
{"msg":"snapshot is oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603@0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:08.27686126Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_06811366-c76f-444f-8d88-46eae8332757/crucible/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603\", \"0bc422a3-9d51-48e9-a8f9-22b4a62ea95f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:08.27688249Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:08.282823179Z","hostname":"oxz_crucible_02080c58-bebd-488b-afe0-a97241b93c7f","pid":26170,"uri":"/crucible/0/regions/40e463cb-af0a-4f90-a80b-9f64aecaf603/snapshots/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f/run","method":"POST","req_id":"29ef059a-198c-47ff-84be-ec18dfe9ab8f","remote_addr":"[fd00:1122:3344:101::c]:41290","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18456,"response_code":200}