[ Aug 14 22:31:10 Enabled. ]
[ Aug 14 22:31:10 Rereading configuration. ]
[ Aug 14 22:31:13 Rereading configuration. ]
[ Aug 14 22:31:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.16234825Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.165314704Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.165327374Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.187438057Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.312896759Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.241880292Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54444"}
{"msg":"region 67bac736-c609-49a4-b920-3138223dbe9c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.264647474Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.270958526Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.275062886Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions","method":"POST","req_id":"7a276473-ad3b-4151-a795-5f55a60bf6bb","remote_addr":"[fd00:1122:3344:101::a]:54444","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16967,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.28286387Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.28288543Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"67bac736-c609-49a4-b920-3138223dbe9c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/67bac736-c609-49a4-b920-3138223dbe9c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.347016647Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"region":"67bac736-c609-49a4-b920-3138223dbe9c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"67bac736-c609-49a4-b920-3138223dbe9c\" \"--data\" \"/data/regions/67bac736-c609-49a4-b920-3138223dbe9c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.348988796Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"region":"67bac736-c609-49a4-b920-3138223dbe9c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.523111229Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions","method":"POST","req_id":"a1bc091f-0c39-4a81-a063-08b34c4e534c","remote_addr":"[fd00:1122:3344:101::a]:54444","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":737,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.55875948Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"region":"67bac736-c609-49a4-b920-3138223dbe9c","component":"worker"}
{"msg":"region 67bac736-c609-49a4-b920-3138223dbe9c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.55879043Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.558923042Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.559385329Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.564015046Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.578938884Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.581905938Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.587836244Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.589945995Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.590016056Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.590066077Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.593845232Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.597417774Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:38.036896372Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions","method":"POST","req_id":"75d27d7f-4145-4742-8810-4ffa97de1384","remote_addr":"[fd00:1122:3344:101::a]:54444","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.328553796Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33127"}
{"msg":"region b10b73de-6533-4ad9-8556-547be441b32b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.328668638Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.329193126Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.329209326Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions","method":"POST","req_id":"b68eda63-b92a-4427-8ed3-79f24b821025","remote_addr":"[fd00:1122:3344:101::a]:33127","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":618,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/b10b73de-6533-4ad9-8556-547be441b32b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.338787486Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/b10b73de-6533-4ad9-8556-547be441b32b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.338826136Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b10b73de-6533-4ad9-8556-547be441b32b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b10b73de-6533-4ad9-8556-547be441b32b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.393704178Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"region":"b10b73de-6533-4ad9-8556-547be441b32b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b10b73de-6533-4ad9-8556-547be441b32b\" \"--data\" \"/data/regions/b10b73de-6533-4ad9-8556-547be441b32b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.393752049Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"region":"b10b73de-6533-4ad9-8556-547be441b32b","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.455838485Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"region":"b10b73de-6533-4ad9-8556-547be441b32b","component":"worker"}
{"msg":"region b10b73de-6533-4ad9-8556-547be441b32b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.455881566Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.455978158Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462112227Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462199578Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.46228077Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.46229075Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.46233488Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462395871Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462496653Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462565664Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462614564Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.462664385Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.47460553Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b10b73de-6533-4ad9-8556-547be441b32b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.475035766Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b10b73de-6533-4ad9-8556-547be441b32b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.484468874Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.492889426Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b10b73de-6533-4ad9-8556-547be441b32b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.495782591Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b10b73de-6533-4ad9-8556-547be441b32b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.496121794Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.496179065Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.496219795Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.496258896Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.499734057Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.503588833Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:01.680758561Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions","method":"POST","req_id":"a18308f3-76da-40bc-9971-8d7b9f52cbfe","remote_addr":"[fd00:1122:3344:101::a]:33127","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.157974175Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions/67bac736-c609-49a4-b920-3138223dbe9c","method":"GET","req_id":"4a29bd3d-671a-4dd4-a727-c7e98bb03092","remote_addr":"[fd00:1122:3344:101::a]:33127","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":611,"response_code":"200"}
{"msg":"path is \"/data/regions/67bac736-c609-49a4-b920-3138223dbe9c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.16314866Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"dataset is oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.17274564Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"snapshot is oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c@f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.180775006Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c\", \"f199dc8b-a47b-43bd-b61e-cf562c2a2ad7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.180801641Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.191413153Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions/67bac736-c609-49a4-b920-3138223dbe9c/snapshots/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","method":"GET","req_id":"48ef1622-64c8-49ec-9311-b1ded60f4ac0","remote_addr":"[fd00:1122:3344:101::a]:33127","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31023,"response_code":"200"}
{"msg":"path is \"/data/regions/67bac736-c609-49a4-b920-3138223dbe9c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.195894649Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"dataset is oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.204690897Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"snapshot is oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c@f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.212355179Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c\", \"f199dc8b-a47b-43bd-b61e-cf562c2a2ad7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.21239441Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 67bac736-c609-49a4-b920-3138223dbe9c-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.2213206Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"applying SMF actions for region 67bac736-c609-49a4-b920-3138223dbe9c running snapshot f199dc8b-a47b-43bd-b61e-cf562c2a2ad7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.221517783Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.221557203Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions/67bac736-c609-49a4-b920-3138223dbe9c/snapshots/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7/run","method":"POST","req_id":"0358e153-2d17-4645-8e7f-f0dc3a2f7f89","remote_addr":"[fd00:1122:3344:101::a]:33127","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25689,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222144162Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222225643Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222303914Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222310354Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222349365Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222393396Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222762301Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222821532Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222868192Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.222915013Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.226681758Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-67bac736-c609-49a4-b920-3138223dbe9c-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.228245611Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-67bac736-c609-49a4-b920-3138223dbe9c-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.234147787Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-67bac736-c609-49a4-b920-3138223dbe9c-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.248156372Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/67bac736-c609-49a4-b920-3138223dbe9c/.zfs/snapshot/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.248464876Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.248538017Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.248583868Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.248624449Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.248684589Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.252336493Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.256427893Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"worker"}
{"msg":"region 67bac736-c609-49a4-b920-3138223dbe9c running snapshot f199dc8b-a47b-43bd-b61e-cf562c2a2ad7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.256443133Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"path is \"/data/regions/67bac736-c609-49a4-b920-3138223dbe9c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.519429234Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"dataset is oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.528297233Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"datafile"}
{"msg":"snapshot is oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c@f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.538657295Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_20226c3a-235f-4118-b164-e14967848f5c/crucible/regions/67bac736-c609-49a4-b920-3138223dbe9c\", \"f199dc8b-a47b-43bd-b61e-cf562c2a2ad7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.538690435Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.547780278Z","hostname":"oxz_crucible_37279933-d27e-4265-9e70-d58d0575c79d","pid":24415,"uri":"/crucible/0/regions/67bac736-c609-49a4-b920-3138223dbe9c/snapshots/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7/run","method":"POST","req_id":"53a701dd-0114-48f2-adb7-60584b23bcfe","remote_addr":"[fd00:1122:3344:101::a]:33127","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":28387,"response_code":"200"}