[ Oct 24 18:54:35 Enabled. ]
[ Oct 24 18:54:35 Rereading configuration. ]
[ Oct 24 18:54:39 Rereading configuration. ]
[ Oct 24 18:54:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.267541248Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.284561641Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.284578761Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.284905466Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.447210858Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.36126489Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59407"}
{"msg":"region 9c8d3953-09d9-408a-ac11-91765ef3b4fa state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.402544763Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.411823556Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions","method":"POST","req_id":"449d897f-4187-439a-bf73-4d6a61537a4e","remote_addr":"[fd00:1122:3344:101::a]:59407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19320,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.411850616Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.421270852Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.421306112Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9c8d3953-09d9-408a-ac11-91765ef3b4fa\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.466855674Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"region":"9c8d3953-09d9-408a-ac11-91765ef3b4fa","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9c8d3953-09d9-408a-ac11-91765ef3b4fa\" \"--data\" \"/data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.471366944Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"region":"9c8d3953-09d9-408a-ac11-91765ef3b4fa","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.553835485Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions","method":"POST","req_id":"421bb23c-ab7e-462b-8cca-d85876b75f04","remote_addr":"[fd00:1122:3344:101::a]:59407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.729597173Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"region":"9c8d3953-09d9-408a-ac11-91765ef3b4fa","component":"worker"}
{"msg":"region 9c8d3953-09d9-408a-ac11-91765ef3b4fa state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.729642354Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.729743156Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.732569739Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.738202656Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.75533711Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.758346607Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.766058846Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.766150637Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.766210238Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.766261959Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.769463848Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.774050299Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:02.256019198Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions","method":"POST","req_id":"65b0709c-6474-4533-a5fd-f133f551acae","remote_addr":"[fd00:1122:3344:101::a]:59407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.281260021Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57387"}
{"msg":"region a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.281400863Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.282052313Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.282069114Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions","method":"POST","req_id":"676dbbff-90e3-428d-a5c6-e2f723f9770d","remote_addr":"[fd00:1122:3344:101::a]:57387","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":699,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.291779643Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.291826184Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.353119289Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"region":"a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8\" \"--data\" \"/data/regions/a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.35318218Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"region":"a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.458654806Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"region":"a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","component":"worker"}
{"msg":"region a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.458696536Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.458777778Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467478412Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467566143Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467644874Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467657485Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467697385Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467762616Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467864408Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.467934029Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.46798218Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.46803664Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.481026781Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.481493498Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.487730734Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.501535427Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.510516985Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a1c3ec38-0581-46a3-b9ed-757e4e8ad2f8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.510876781Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.510945402Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.510997413Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.511046783Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.516913884Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.524087215Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:54.548571592Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions","method":"POST","req_id":"c9aab780-23e6-4094-9900-9858dd72d8a4","remote_addr":"[fd00:1122:3344:101::a]:57387","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.337126527Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","method":"GET","req_id":"8deb7b2d-9733-4e69-afe0-a1b30cf3cb35","remote_addr":"[fd00:1122:3344:101::a]:57387","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":3930,"response_code":"200"}
{"msg":"path is \"/data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.340364121Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"dataset is oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.349768972Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"snapshot is oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa@63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.358110241Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\", \"63eefcdc-6ccb-408c-aa68-17bee2851ce7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.358147281Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.371126261Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa/snapshots/63eefcdc-6ccb-408c-aa68-17bee2851ce7","method":"GET","req_id":"0fbd4cda-217a-458d-934e-8910bd0098d2","remote_addr":"[fd00:1122:3344:101::a]:57387","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":30702,"response_code":"200"}
{"msg":"path is \"/data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.374384632Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"dataset is oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.382893773Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"snapshot is oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa@63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.390992618Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\", \"63eefcdc-6ccb-408c-aa68-17bee2851ce7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.391035928Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 9c8d3953-09d9-408a-ac11-91765ef3b4fa-63eefcdc-6ccb-408c-aa68-17bee2851ce7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.401130124Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"applying SMF actions for region 9c8d3953-09d9-408a-ac11-91765ef3b4fa running snapshot 63eefcdc-6ccb-408c-aa68-17bee2851ce7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.401317537Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.401387898Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa/snapshots/63eefcdc-6ccb-408c-aa68-17bee2851ce7/run","method":"POST","req_id":"b73f115e-37c9-454b-9489-25c9ce32306a","remote_addr":"[fd00:1122:3344:101::a]:57387","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27479,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.401907766Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.401992667Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402074678Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402081398Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402122379Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.4021687Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402562076Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402622627Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402671388Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.402717268Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.410918345Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-9c8d3953-09d9-408a-ac11-91765ef3b4fa-63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.412457118Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-9c8d3953-09d9-408a-ac11-91765ef3b4fa-63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.420860078Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-9c8d3953-09d9-408a-ac11-91765ef3b4fa-63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.442046035Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa/.zfs/snapshot/63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.44239555Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.442476131Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.442547582Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.442608983Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.442690724Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.451734274Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.461570006Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"worker"}
{"msg":"region 9c8d3953-09d9-408a-ac11-91765ef3b4fa running snapshot 63eefcdc-6ccb-408c-aa68-17bee2851ce7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.461590106Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"path is \"/data/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.64933027Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"dataset is oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.659142911Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"datafile"}
{"msg":"snapshot is oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa@63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.66750825Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2cb5d21f-d22a-413f-a49c-154a4b1cfaf9/crucible/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa\", \"63eefcdc-6ccb-408c-aa68-17bee2851ce7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.66754413Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:55.675583664Z","hostname":"oxz_crucible_7d0cdafd-c4b9-4211-a037-354ab6337ccd","pid":24011,"uri":"/crucible/0/regions/9c8d3953-09d9-408a-ac11-91765ef3b4fa/snapshots/63eefcdc-6ccb-408c-aa68-17bee2851ce7/run","method":"POST","req_id":"965c0693-bcdc-481b-a52e-e4a719b479c9","remote_addr":"[fd00:1122:3344:101::a]:57387","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26306,"response_code":"200"}