[ Dec 20 04:54:04 Enabled. ]
[ Dec 20 04:54:04 Rereading configuration. ]
[ Dec 20 04:54:07 Rereading configuration. ]
[ Dec 20 04:54:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c12e9b05-5016-4fc0-83c4-d84ad0d50d46/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:13.682432119Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838}
{"msg":"dataset: \"oxp_c12e9b05-5016-4fc0-83c4-d84ad0d50d46/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:13.683367143Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:13.683378463Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:13.683384313Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:13.696114687Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:13.803132715Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.749132633Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:46697"}
{"msg":"region 134bf708-935f-46e4-a144-71c0f109b799 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.792842948Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.795737232Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.795773302Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions","method":"POST","req_id":"bf0044ab-19ad-4bd0-9700-f22542dfc15b","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18828,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_c12e9b05-5016-4fc0-83c4-d84ad0d50d46/crucible/regions/134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.806636748Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_c12e9b05-5016-4fc0-83c4-d84ad0d50d46/crucible/regions/134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.806653908Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"134bf708-935f-46e4-a144-71c0f109b799\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/134bf708-935f-46e4-a144-71c0f109b799\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.869827889Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"region":"134bf708-935f-46e4-a144-71c0f109b799","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"134bf708-935f-46e4-a144-71c0f109b799\" \"--data\" \"/data/regions/134bf708-935f-46e4-a144-71c0f109b799\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.872744224Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"region":"134bf708-935f-46e4-a144-71c0f109b799","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.119130813Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions","method":"POST","req_id":"3cfeff79-9bad-4e97-a58d-21e6554204af","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.622149497Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"region":"134bf708-935f-46e4-a144-71c0f109b799","component":"worker"}
{"msg":"region 134bf708-935f-46e4-a144-71c0f109b799 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.622204838Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.622282569Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.622630764Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.746218405Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.848471641Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions","method":"POST","req_id":"7a6f679f-34dd-4729-893e-c2a1235de6c6","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.888792035Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.934318838Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.934993318Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.935069529Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.93511942Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.935170031Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.971323151Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.996862269Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.152360719Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"GET","req_id":"5ce39fbb-62ce-4243-95e3-c4aeb04a926c","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":86,"response_code":200}
{"msg":"path is \"/data/regions/134bf708-935f-46e4-a144-71c0f109b799\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.1576777Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"dataset is oxp_c12e9b05-5016-4fc0-83c4-d84ad0d50d46/crucible/regions/134bf708-935f-46e4-a144-71c0f109b799","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.165476919Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"region 134bf708-935f-46e4-a144-71c0f109b799 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.172196901Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.172326573Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"DELETE","req_id":"fd4966ee-496e-4c5d-be8d-66a52c0ebb47","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14672,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.172347593Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-134bf708-935f-46e4-a144-71c0f109b799 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.17342698Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.178317514Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"GET","req_id":"f12109d2-cb1a-46a0-aa8f-56e9fcf9ee2f","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.212939341Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_c12e9b05-5016-4fc0-83c4-d84ad0d50d46/crucible/regions/134bf708-935f-46e4-a144-71c0f109b799\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.22011446Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"region":"134bf708-935f-46e4-a144-71c0f109b799","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.251702881Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"GET","req_id":"6a55f221-a976-45a3-99b3-87aeeadb0913","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":77,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.372062533Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"GET","req_id":"96069170-2ff6-48b5-bec5-e9fd6d8a771a","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.45211482Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"GET","req_id":"2bd83429-5af4-4681-a20f-5d8b420528b2","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":28,"response_code":200}
{"msg":"region 134bf708-935f-46e4-a144-71c0f109b799 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.638582678Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.650528569Z","hostname":"oxz_crucible_33ca0dcf-50dc-425e-8b2b-4ad8ace3602b","pid":22838,"uri":"/crucible/0/regions/134bf708-935f-46e4-a144-71c0f109b799","method":"GET","req_id":"8c79dedb-aba8-4736-a605-cf8d2bfca16d","remote_addr":"[fd00:1122:3344:101::c]:46697","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":43,"response_code":200}