[ Jul  6 00:07:51 Enabled. ]
[ Jul  6 00:07:51 Rereading configuration. ]
[ Jul  6 00:07:53 Rereading configuration. ]
[ Jul  6 00:07:55 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:07:55.690413643Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:07:55.695081738Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:07:55.695099268Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:07:55.715743077Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:07:55.827185767Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.616053053Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35991"}
{"msg":"region ca1c0728-f767-41ea-af07-b6f3c894768e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.621666682Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.622730099Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"e2046f3f-029f-412c-aad6-c5b2f3e1ddab","remote_addr":"[fd00:1122:3344:101::c]:35991","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3660,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.622745109Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.63593281Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.635963671Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ca1c0728-f767-41ea-af07-b6f3c894768e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ca1c0728-f767-41ea-af07-b6f3c894768e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.69415966Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"ca1c0728-f767-41ea-af07-b6f3c894768e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ca1c0728-f767-41ea-af07-b6f3c894768e\" \"--data\" \"/data/regions/ca1c0728-f767-41ea-af07-b6f3c894768e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.694222741Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"ca1c0728-f767-41ea-af07-b6f3c894768e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.81000404Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"ca1c0728-f767-41ea-af07-b6f3c894768e","component":"worker"}
{"msg":"region ca1c0728-f767-41ea-af07-b6f3c894768e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.810062491Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.810140502Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.810610049Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.813939602Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.82254984Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.825547858Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.826108797Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.826234369Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.82628013Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.82631937Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.830932384Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.835114601Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:08:59.931855496Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"c51cb216-2cf4-41d0-9aeb-00df73019b90","remote_addr":"[fd00:1122:3344:101::c]:35991","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.446805128Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","method":"GET","req_id":"1cd44fa1-c3b6-4690-97b1-ba4cbae81b45","remote_addr":"[fd00:1122:3344:101::c]:35991","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"path is \"/data/regions/ca1c0728-f767-41ea-af07-b6f3c894768e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.448880391Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"dataset is oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.457696592Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"region ca1c0728-f767-41ea-af07-b6f3c894768e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.465585967Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.46571932Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.465779201Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","method":"DELETE","req_id":"2380d0db-fea9-4fbb-86b2-9d514361b7d0","remote_addr":"[fd00:1122:3344:101::c]:35991","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":16939,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-ca1c0728-f767-41ea-af07-b6f3c894768e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.466778417Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.468571095Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","method":"GET","req_id":"9048038d-2647-4e35-9b10-f55eeb471b84","remote_addr":"[fd00:1122:3344:101::c]:35991","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.471415561Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/ca1c0728-f767-41ea-af07-b6f3c894768e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.481586693Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"ca1c0728-f767-41ea-af07-b6f3c894768e","component":"worker"}
{"msg":"region ca1c0728-f767-41ea-af07-b6f3c894768e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.535717367Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:00.578646683Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/ca1c0728-f767-41ea-af07-b6f3c894768e","method":"GET","req_id":"5069c2ce-e662-4ed7-aab2-c800514cc3bc","remote_addr":"[fd00:1122:3344:101::c]:35991","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.173345273Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33129"}
{"msg":"region 866e802e-9fe2-4d11-aadf-37b953c9c772 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.173496345Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.17374664Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.17375856Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"94146b95-e3cf-4747-8bf2-2fb44a8a9523","remote_addr":"[fd00:1122:3344:101::a]:33129","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":292,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.183432774Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.183462805Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"866e802e-9fe2-4d11-aadf-37b953c9c772\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/866e802e-9fe2-4d11-aadf-37b953c9c772\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.247911014Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"866e802e-9fe2-4d11-aadf-37b953c9c772","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"866e802e-9fe2-4d11-aadf-37b953c9c772\" \"--data\" \"/data/regions/866e802e-9fe2-4d11-aadf-37b953c9c772\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.25019822Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"866e802e-9fe2-4d11-aadf-37b953c9c772","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.313479171Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"866e802e-9fe2-4d11-aadf-37b953c9c772","component":"worker"}
{"msg":"region 866e802e-9fe2-4d11-aadf-37b953c9c772 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.313506001Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.313640163Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.315172687Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.318782235Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.326271964Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.330011014Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.330293919Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.330347719Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.33039161Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.330432781Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.333588681Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.336155522Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"2fb2fedf-7dbc-4a53-9313-b22ae1c3f5d5","remote_addr":"[fd00:1122:3344:101::a]:33129","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:09:08.338288846Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.582013525Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56130"}
{"msg":"region 0b789859-1ce3-4b9e-8172-a0b2a1024dd9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.582141447Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.582660155Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.582670115Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"d5082784-ea0b-4d13-8d72-2d0fef59b008","remote_addr":"[fd00:1122:3344:101::a]:56130","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":622,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/0b789859-1ce3-4b9e-8172-a0b2a1024dd9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.593640901Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/0b789859-1ce3-4b9e-8172-a0b2a1024dd9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.593679751Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0b789859-1ce3-4b9e-8172-a0b2a1024dd9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0b789859-1ce3-4b9e-8172-a0b2a1024dd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.655725242Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"0b789859-1ce3-4b9e-8172-a0b2a1024dd9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0b789859-1ce3-4b9e-8172-a0b2a1024dd9\" \"--data\" \"/data/regions/0b789859-1ce3-4b9e-8172-a0b2a1024dd9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.655784753Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"0b789859-1ce3-4b9e-8172-a0b2a1024dd9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.726077865Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"region":"0b789859-1ce3-4b9e-8172-a0b2a1024dd9","component":"worker"}
{"msg":"region 0b789859-1ce3-4b9e-8172-a0b2a1024dd9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.726123836Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.726246208Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0b789859-1ce3-4b9e-8172-a0b2a1024dd9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.727303005Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0b789859-1ce3-4b9e-8172-a0b2a1024dd9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.736785516Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.751067775Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0b789859-1ce3-4b9e-8172-a0b2a1024dd9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.753888969Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0b789859-1ce3-4b9e-8172-a0b2a1024dd9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.754186804Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.754243105Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.754284456Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.754322996Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.757463657Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.762745291Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:29.909047487Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"813f0f06-fcf2-4ca0-9d53-f6e3b8b6494a","remote_addr":"[fd00:1122:3344:101::a]:56130","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":112,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.948713727Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47541"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.948776338Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/866e802e-9fe2-4d11-aadf-37b953c9c772","method":"GET","req_id":"0efefdf4-9f8b-423f-adf2-1323de1841eb","remote_addr":"[fd00:1122:3344:101::a]:47541","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"path is \"/data/regions/866e802e-9fe2-4d11-aadf-37b953c9c772\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.948984962Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"dataset is oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.957922144Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"snapshot is oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772@3916d487-df2b-42eb-b478-d4b46764e516","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.967232923Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772\", \"3916d487-df2b-42eb-b478-d4b46764e516\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.967274674Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.978634825Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/866e802e-9fe2-4d11-aadf-37b953c9c772/snapshots/3916d487-df2b-42eb-b478-d4b46764e516","method":"GET","req_id":"4096cd24-2692-464b-93e8-93d5d6f96dcd","remote_addr":"[fd00:1122:3344:101::a]:47541","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29693,"response_code":"200"}
{"msg":"path is \"/data/regions/866e802e-9fe2-4d11-aadf-37b953c9c772\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.978818718Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"dataset is oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.9877008Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"snapshot is oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772@3916d487-df2b-42eb-b478-d4b46764e516","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.995986312Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crucible/regions/866e802e-9fe2-4d11-aadf-37b953c9c772\", \"3916d487-df2b-42eb-b478-d4b46764e516\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:30.996025133Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 866e802e-9fe2-4d11-aadf-37b953c9c772-3916d487-df2b-42eb-b478-d4b46764e516 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.008159947Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}
{"msg":"applying SMF actions for region 866e802e-9fe2-4d11-aadf-37b953c9c772 running snapshot 3916d487-df2b-42eb-b478-d4b46764e516 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.008309489Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.008350339Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"uri":"/crucible/0/regions/866e802e-9fe2-4d11-aadf-37b953c9c772/snapshots/3916d487-df2b-42eb-b478-d4b46764e516/run","method":"POST","req_id":"e1625e7f-6d65-4985-98d8-48bbc5b28870","remote_addr":"[fd00:1122:3344:101::a]:47541","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29566,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-866e802e-9fe2-4d11-aadf-37b953c9c772-3916d487-df2b-42eb-b478-d4b46764e516","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.011351627Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-866e802e-9fe2-4d11-aadf-37b953c9c772-3916d487-df2b-42eb-b478-d4b46764e516","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.014124232Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-866e802e-9fe2-4d11-aadf-37b953c9c772-3916d487-df2b-42eb-b478-d4b46764e516","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.023305688Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/866e802e-9fe2-4d11-aadf-37b953c9c772/.zfs/snapshot/3916d487-df2b-42eb-b478-d4b46764e516","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.023537922Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.023597293Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.023647594Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.023689595Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.023742125Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.02715107Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.03157917Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"worker"}
{"msg":"region 866e802e-9fe2-4d11-aadf-37b953c9c772 running snapshot 3916d487-df2b-42eb-b478-d4b46764e516 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-06T00:14:31.031589601Z","hostname":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","pid":25684,"component":"datafile"}