[ Jul 23 07:46:40 Enabled. ]
[ Jul 23 07:46:40 Rereading configuration. ]
[ Jul 23 07:46:41 Rereading configuration. ]
[ Jul 23 07:46:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.166568956Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.169788914Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.169800234Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.18568559Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.352732377Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.259824849Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43053"}
{"msg":"region 5f2d0713-5d52-434c-b23b-fe11aae16603 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.285006504Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.289952217Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"f967dfa6-626b-454f-a788-67503370e2d3","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19587,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.291983854Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.303155689Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.30322512Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5f2d0713-5d52-434c-b23b-fe11aae16603\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5f2d0713-5d52-434c-b23b-fe11aae16603\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.377737676Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"5f2d0713-5d52-434c-b23b-fe11aae16603","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5f2d0713-5d52-434c-b23b-fe11aae16603\" \"--data\" \"/data/regions/5f2d0713-5d52-434c-b23b-fe11aae16603\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.380056196Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"5f2d0713-5d52-434c-b23b-fe11aae16603","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.532025855Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"a43894b8-b069-43b4-bd2c-53ff7423f4aa","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.690047835Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"5f2d0713-5d52-434c-b23b-fe11aae16603","component":"worker"}
{"msg":"region 5f2d0713-5d52-434c-b23b-fe11aae16603 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.690138146Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.690346777Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.690730981Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.695863214Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.713571176Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.716982695Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.717679151Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.717742601Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.717784732Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.717821142Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.721825186Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:07.735900997Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.248655187Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"d96889b1-ba8d-4bd8-9b8c-b8bbddfb3173","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.747269928Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","method":"GET","req_id":"0922aa24-d6c5-4bab-8609-d1c1892f7113","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"path is \"/data/regions/5f2d0713-5d52-434c-b23b-fe11aae16603\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.750789329Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"dataset is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.761679451Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"region 5f2d0713-5d52-434c-b23b-fe11aae16603 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.772780355Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.772977087Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.773046388Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","method":"DELETE","req_id":"fff4eeba-eec2-4b71-9fd8-ad2f0f038177","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22254,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-5f2d0713-5d52-434c-b23b-fe11aae16603 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.774027666Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.776549978Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","method":"GET","req_id":"32211124-e601-4d15-a705-a449061ef6a1","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.778715006Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/5f2d0713-5d52-434c-b23b-fe11aae16603\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.790127344Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"5f2d0713-5d52-434c-b23b-fe11aae16603","component":"worker"}
{"msg":"region 5f2d0713-5d52-434c-b23b-fe11aae16603 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.870934734Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:08.925340089Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/5f2d0713-5d52-434c-b23b-fe11aae16603","method":"GET","req_id":"c724bd0a-3166-44bc-9ea5-e77cba06df37","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"region 0794267e-6fbc-470a-88ff-de23b89b6119 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.110353317Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.110550329Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"0d016e4f-1627-49c3-83a3-e5d3994db84e","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":260,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.11060148Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.119159462Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.119187403Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0794267e-6fbc-470a-88ff-de23b89b6119\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0794267e-6fbc-470a-88ff-de23b89b6119\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.190866715Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"0794267e-6fbc-470a-88ff-de23b89b6119","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0794267e-6fbc-470a-88ff-de23b89b6119\" \"--data\" \"/data/regions/0794267e-6fbc-470a-88ff-de23b89b6119\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.192712421Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"0794267e-6fbc-470a-88ff-de23b89b6119","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.265585303Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"0794267e-6fbc-470a-88ff-de23b89b6119","component":"worker"}
{"msg":"region 0794267e-6fbc-470a-88ff-de23b89b6119 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.265658404Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.265864126Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.267165427Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.270254733Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.278139871Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.281190857Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.28150378Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.28155539Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.28159513Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.281635141Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.285582974Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.290093336Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:47:11.391106296Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"fc4b2f43-af01-477f-bed6-b254ec38edec","remote_addr":"[fd00:1122:3344:101::c]:43053","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.936784951Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61717"}
{"msg":"region 062e1c68-c96d-4322-89bd-53b86bba1392 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.937485067Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.937586268Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"47cb26b0-8a61-4f6c-9293-ebc60445365f","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":714,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.937604458Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/062e1c68-c96d-4322-89bd-53b86bba1392","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.949634731Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/062e1c68-c96d-4322-89bd-53b86bba1392","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.949675751Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"062e1c68-c96d-4322-89bd-53b86bba1392\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/062e1c68-c96d-4322-89bd-53b86bba1392\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.010102977Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"062e1c68-c96d-4322-89bd-53b86bba1392","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"062e1c68-c96d-4322-89bd-53b86bba1392\" \"--data\" \"/data/regions/062e1c68-c96d-4322-89bd-53b86bba1392\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.012120885Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"062e1c68-c96d-4322-89bd-53b86bba1392","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.080102815Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"9e1eabf7-0b1d-40cd-be5c-e34bcabf742a","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.141974324Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"region":"062e1c68-c96d-4322-89bd-53b86bba1392","component":"worker"}
{"msg":"region 062e1c68-c96d-4322-89bd-53b86bba1392 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.142001424Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.142139455Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-062e1c68-c96d-4322-89bd-53b86bba1392","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.143015593Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-062e1c68-c96d-4322-89bd-53b86bba1392","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.159568654Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.180784906Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-062e1c68-c96d-4322-89bd-53b86bba1392","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.183945102Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/062e1c68-c96d-4322-89bd-53b86bba1392","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.184234975Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.184281096Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.184318416Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.184354956Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.187976577Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.193740776Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.536419139Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions","method":"POST","req_id":"0825136a-655e-42fd-a5bf-22e2e194eb54","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.57534391Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/0794267e-6fbc-470a-88ff-de23b89b6119","method":"GET","req_id":"7216268d-c551-4a0f-9572-8eb0630f8eb1","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"path is \"/data/regions/0794267e-6fbc-470a-88ff-de23b89b6119\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.57882302Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"dataset is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.587722026Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"snapshot is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119@88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.596450961Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119\", \"88d026d8-f097-43a2-b27e-5132183fe289\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.596478841Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.607596636Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/0794267e-6fbc-470a-88ff-de23b89b6119/snapshots/88d026d8-f097-43a2-b27e-5132183fe289","method":"GET","req_id":"644a8a64-f95f-4f6b-99a4-07a4220e4d7a","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":28745,"response_code":"200"}
{"msg":"path is \"/data/regions/0794267e-6fbc-470a-88ff-de23b89b6119\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.61165496Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"dataset is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.620426406Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"snapshot is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119@88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.629691605Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119\", \"88d026d8-f097-43a2-b27e-5132183fe289\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.629728855Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 0794267e-6fbc-470a-88ff-de23b89b6119-88d026d8-f097-43a2-b27e-5132183fe289 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.640389016Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"applying SMF actions for region 0794267e-6fbc-470a-88ff-de23b89b6119 running snapshot 88d026d8-f097-43a2-b27e-5132183fe289 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.640582618Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.640601548Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/0794267e-6fbc-470a-88ff-de23b89b6119/snapshots/88d026d8-f097-43a2-b27e-5132183fe289/run","method":"POST","req_id":"ef3b4ea5-56b1-40ce-8fa7-b9486cfc5348","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":28975,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-0794267e-6fbc-470a-88ff-de23b89b6119-88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.643110939Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-0794267e-6fbc-470a-88ff-de23b89b6119-88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.646198216Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-0794267e-6fbc-470a-88ff-de23b89b6119-88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.656348392Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0794267e-6fbc-470a-88ff-de23b89b6119/.zfs/snapshot/88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.656576284Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.656627335Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.656668965Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.656705096Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.656750256Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.660269086Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.664347631Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"worker"}
{"msg":"region 0794267e-6fbc-470a-88ff-de23b89b6119 running snapshot 88d026d8-f097-43a2-b27e-5132183fe289 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:54.664366691Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"path is \"/data/regions/0794267e-6fbc-470a-88ff-de23b89b6119\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:55.019370674Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"dataset is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:55.028683604Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"datafile"}
{"msg":"snapshot is oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119@88d026d8-f097-43a2-b27e-5132183fe289","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:55.037266297Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5f7be984-ff66-4ddb-bfd4-6f177ea7dfa1/crucible/regions/0794267e-6fbc-470a-88ff-de23b89b6119\", \"88d026d8-f097-43a2-b27e-5132183fe289\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:55.037312957Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:55.045256325Z","hostname":"oxz_crucible_6519ca38-1b68-47a4-824e-c4049183aa0e","pid":26548,"uri":"/crucible/0/regions/0794267e-6fbc-470a-88ff-de23b89b6119/snapshots/88d026d8-f097-43a2-b27e-5132183fe289/run","method":"POST","req_id":"2d1bc04d-9d0f-4a67-aa84-64fba3cd4012","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25898,"response_code":"200"}