[ Feb 21 01:16:56 Enabled. ]
[ Feb 21 01:16:56 Rereading configuration. ]
[ Feb 21 01:16:58 Rereading configuration. ]
[ Feb 21 01:17:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:17:00.164232386Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:17:00.165330679Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:17:00.165340539Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:17:00.296217738Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.173273699Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33276"}
{"msg":"region 5ddcb8d4-bed4-487d-9a77-98053c37874e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.252165441Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.254972585Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions","method":"POST","req_id":"e2bc71de-bc9d-4d33-92cc-528ad3416f04","remote_addr":"[fd00:1122:3344:101::a]:33276","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":14079,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.255002836Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.263440429Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.26346993Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5ddcb8d4-bed4-487d-9a77-98053c37874e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.321275811Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"region":"5ddcb8d4-bed4-487d-9a77-98053c37874e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.382949348Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions","method":"POST","req_id":"549082c9-9524-4836-981a-8bc20ede4f22","remote_addr":"[fd00:1122:3344:101::a]:33276","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":624,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.447665886Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"region":"5ddcb8d4-bed4-487d-9a77-98053c37874e","component":"worker"}
{"msg":"region 5ddcb8d4-bed4-487d-9a77-98053c37874e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.447699767Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.4478265Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.448293984Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.455314095Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.465458058Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.467955563Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.478340414Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.480349034Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.480438477Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.480510059Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.485049695Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.488353374Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:18:34.933805433Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions","method":"POST","req_id":"b1638d9c-dd41-4aa8-863a-bb31dcdbde27","remote_addr":"[fd00:1122:3344:101::a]:33276","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:56.983841413Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64239"}
{"msg":"region 94088b25-4202-488a-af76-4a4a0d5ace14 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.013718758Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.01878655Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.01880781Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions","method":"POST","req_id":"13e7debc-912b-4ff3-a6cb-82f11570f4cf","remote_addr":"[fd00:1122:3344:101::a]:64239","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":5109,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/94088b25-4202-488a-af76-4a4a0d5ace14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.028672666Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/94088b25-4202-488a-af76-4a4a0d5ace14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.028710617Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"94088b25-4202-488a-af76-4a4a0d5ace14\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/94088b25-4202-488a-af76-4a4a0d5ace14\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.147872255Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"region":"94088b25-4202-488a-af76-4a4a0d5ace14","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.187539233Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions","method":"POST","req_id":"e52bd366-74f3-45b4-beac-d1b40d620a4c","remote_addr":"[fd00:1122:3344:101::a]:64239","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.480823466Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"region":"94088b25-4202-488a-af76-4a4a0d5ace14","component":"worker"}
{"msg":"region 94088b25-4202-488a-af76-4a4a0d5ace14 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.480859907Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.481041952Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.484915508Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485001871Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485083864Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485098164Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485132085Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485182526Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485279969Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485349111Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485399113Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.485453484Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.507546106Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-94088b25-4202-488a-af76-4a4a0d5ace14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.508063141Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-94088b25-4202-488a-af76-4a4a0d5ace14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.54406334Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.583725027Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94088b25-4202-488a-af76-4a4a0d5ace14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.607628833Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94088b25-4202-488a-af76-4a4a0d5ace14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.607881191Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.607949643Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.607998704Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.608047156Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.632347593Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.64427282Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:57.925177492Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions","method":"POST","req_id":"b1a7689d-acea-4baa-a8f5-9a3d61cb5093","remote_addr":"[fd00:1122:3344:101::a]:64239","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.078585472Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49958"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.083313443Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","method":"GET","req_id":"ef209d1a-ccbf-4c64-81ec-f4ffa21e6fb0","remote_addr":"[fd00:1122:3344:101::a]:49958","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":4708,"response_code":"200"}
{"msg":"path is \"/data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.083463488Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.093080086Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e@a3e9b599-0b1b-4d20-be9f-3b9142328a61","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.101119606Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e\", \"a3e9b599-0b1b-4d20-be9f-3b9142328a61\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.101160868Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.121835287Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e/snapshots/a3e9b599-0b1b-4d20-be9f-3b9142328a61","method":"GET","req_id":"9c484c7f-68e9-4731-92a9-a2520a072b26","remote_addr":"[fd00:1122:3344:101::a]:49958","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":38380,"response_code":"200"}
{"msg":"path is \"/data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.122006372Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.130378992Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e@a3e9b599-0b1b-4d20-be9f-3b9142328a61","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.138550386Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e\", \"a3e9b599-0b1b-4d20-be9f-3b9142328a61\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.138586278Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5ddcb8d4-bed4-487d-9a77-98053c37874e-a3e9b599-0b1b-4d20-be9f-3b9142328a61 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.148228746Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}
{"msg":"applying SMF actions for region 5ddcb8d4-bed4-487d-9a77-98053c37874e running snapshot a3e9b599-0b1b-4d20-be9f-3b9142328a61 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.148377581Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.148426292Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"uri":"/crucible/0/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e/snapshots/a3e9b599-0b1b-4d20-be9f-3b9142328a61/run","method":"POST","req_id":"07945643-e543-41d4-bd1c-5bc68c27c268","remote_addr":"[fd00:1122:3344:101::a]:49958","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26443,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.148944658Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.14903744Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149127883Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149134763Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149180485Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149228026Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149569747Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149628958Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.14967585Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.149721461Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.172548685Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-5ddcb8d4-bed4-487d-9a77-98053c37874e-a3e9b599-0b1b-4d20-be9f-3b9142328a61","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.174085351Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5ddcb8d4-bed4-487d-9a77-98053c37874e-a3e9b599-0b1b-4d20-be9f-3b9142328a61","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.207047518Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5ddcb8d4-bed4-487d-9a77-98053c37874e-a3e9b599-0b1b-4d20-be9f-3b9142328a61","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.283355353Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5ddcb8d4-bed4-487d-9a77-98053c37874e/.zfs/snapshot/a3e9b599-0b1b-4d20-be9f-3b9142328a61","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.283740964Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.283833717Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.283884439Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.28393024Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.283991332Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.307421083Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.333741432Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"worker"}
{"msg":"region 5ddcb8d4-bed4-487d-9a77-98053c37874e running snapshot a3e9b599-0b1b-4d20-be9f-3b9142328a61 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-21T01:23:59.333772403Z","hostname":"oxz_crucible_da803974-d01d-444d-9603-508c8c6b4ce8","pid":24039,"component":"datafile"}