[ Jan  8 01:15:32 Enabled. ]
[ Jan  8 01:15:32 Rereading configuration. ]
[ Jan  8 01:15:34 Rereading configuration. ]
[ Jan  8 01:15:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:38.907495953Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:38.911212701Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:38.911226552Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:38.931863707Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:39.011474181Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.685322452Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60550"}
{"msg":"region 1e06e39f-f5a0-4912-90da-75173f58a5d4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.70547095Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.707548862Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.712352488Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"963ff27b-6ce5-49cb-af4d-bc62e9404bb1","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":8968,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.71754694Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.71756698Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1e06e39f-f5a0-4912-90da-75173f58a5d4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.77215917Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"1e06e39f-f5a0-4912-90da-75173f58a5d4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1e06e39f-f5a0-4912-90da-75173f58a5d4\" \"--data\" \"/data/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.77410328Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"1e06e39f-f5a0-4912-90da-75173f58a5d4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.876025886Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"5a4e3451-1db6-4d90-af95-5af6029ceeca","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.886786165Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"1e06e39f-f5a0-4912-90da-75173f58a5d4","component":"worker"}
{"msg":"region 1e06e39f-f5a0-4912-90da-75173f58a5d4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.886817286Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.886954898Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.887407205Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.891446909Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.902789467Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.905792654Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.906522226Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.906580997Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.906624678Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.906663588Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.912241466Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:25.916617405Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.608612034Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"44899783-4f2c-4078-aace-c5b5def4ca57","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.961824807Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","method":"GET","req_id":"af0ff27f-a2db-47aa-986f-5ba6356066f7","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"path is \"/data/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.966222136Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"dataset is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.977665576Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"region 1e06e39f-f5a0-4912-90da-75173f58a5d4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.988940014Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.989076456Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.989160357Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","method":"DELETE","req_id":"cd0e2f73-451a-472a-a051-e091b9918d4f","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22989,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-1e06e39f-f5a0-4912-90da-75173f58a5d4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.990353536Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.994949699Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","method":"GET","req_id":"59464a90-92b2-4b2e-a790-b661e5d71ff0","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:26.995185443Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:27.007266363Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"1e06e39f-f5a0-4912-90da-75173f58a5d4","component":"worker"}
{"msg":"region 1e06e39f-f5a0-4912-90da-75173f58a5d4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:27.072166504Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:27.149198717Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/1e06e39f-f5a0-4912-90da-75173f58a5d4","method":"GET","req_id":"5c586f61-e6b0-4bf6-b591-765b52d98aea","remote_addr":"[fd00:1122:3344:101::c]:60550","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.831538343Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33770"}
{"msg":"region f0001f13-0a78-4e17-bdab-035b3367891b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.831741696Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.831905778Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.831913878Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"f8a136bd-3110-425a-bd20-3796e33a250a","remote_addr":"[fd00:1122:3344:101::a]:33770","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":221,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.841576751Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.841606831Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f0001f13-0a78-4e17-bdab-035b3367891b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f0001f13-0a78-4e17-bdab-035b3367891b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.904934649Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"f0001f13-0a78-4e17-bdab-035b3367891b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f0001f13-0a78-4e17-bdab-035b3367891b\" \"--data\" \"/data/regions/f0001f13-0a78-4e17-bdab-035b3367891b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.904988649Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"f0001f13-0a78-4e17-bdab-035b3367891b","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.975827975Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"f0001f13-0a78-4e17-bdab-035b3367891b","component":"worker"}
{"msg":"region f0001f13-0a78-4e17-bdab-035b3367891b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.975865056Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.975951997Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.977475001Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.980290655Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.994187305Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.998685275Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.998884419Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.998939749Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.99898143Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:48.999020461Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:49.002269662Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:49.006897295Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:16:49.166090291Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"d6425f0e-2375-4311-98fd-d9a5a2de1968","remote_addr":"[fd00:1122:3344:101::a]:33770","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.05808152Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41446"}
{"msg":"region 4ca496aa-2730-4c51-bd00-dba9bfbd8c6d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.058251133Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.058860632Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.058870683Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"f0e5b520-c351-4686-a912-44658fc7bc7a","remote_addr":"[fd00:1122:3344:101::a]:41446","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":676,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.068489104Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.068519484Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4ca496aa-2730-4c51-bd00-dba9bfbd8c6d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4ca496aa-2730-4c51-bd00-dba9bfbd8c6d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.1183608Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4ca496aa-2730-4c51-bd00-dba9bfbd8c6d\" \"--data\" \"/data/regions/4ca496aa-2730-4c51-bd00-dba9bfbd8c6d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.11841523Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.196330108Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"region":"4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","component":"worker"}
{"msg":"region 4ca496aa-2730-4c51-bd00-dba9bfbd8c6d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.196376668Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.19648033Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.197532416Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.209481635Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.219749646Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.222912046Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4ca496aa-2730-4c51-bd00-dba9bfbd8c6d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.223221831Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.223272502Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.223315183Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.223351293Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.227094552Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.232948284Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.350722699Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions","method":"POST","req_id":"4c95fe81-1e1f-4e21-a0c6-19f223647889","remote_addr":"[fd00:1122:3344:101::a]:41446","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:14.991532042Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/f0001f13-0a78-4e17-bdab-035b3367891b","method":"GET","req_id":"0f9e2065-8f12-4ff2-8459-f037aaa89b53","remote_addr":"[fd00:1122:3344:101::a]:41446","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":637,"response_code":"200"}
{"msg":"path is \"/data/regions/f0001f13-0a78-4e17-bdab-035b3367891b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:14.994983437Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"dataset is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.00473462Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"snapshot is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b@cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.013654801Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b\", \"cfc9d963-cb47-49ce-bf1c-ee0920dc9aff\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.013697571Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.024634454Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/f0001f13-0a78-4e17-bdab-035b3367891b/snapshots/cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","method":"GET","req_id":"35acabf0-1f70-4170-a8cb-1799f470cf1f","remote_addr":"[fd00:1122:3344:101::a]:41446","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29667,"response_code":"200"}
{"msg":"path is \"/data/regions/f0001f13-0a78-4e17-bdab-035b3367891b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.028114048Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"dataset is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.036426789Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"snapshot is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b@cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.044970824Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b\", \"cfc9d963-cb47-49ce-bf1c-ee0920dc9aff\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.044997604Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f0001f13-0a78-4e17-bdab-035b3367891b-cfc9d963-cb47-49ce-bf1c-ee0920dc9aff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.053515988Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"applying SMF actions for region f0001f13-0a78-4e17-bdab-035b3367891b running snapshot cfc9d963-cb47-49ce-bf1c-ee0920dc9aff (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.053689551Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.053725752Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/f0001f13-0a78-4e17-bdab-035b3367891b/snapshots/cfc9d963-cb47-49ce-bf1c-ee0920dc9aff/run","method":"POST","req_id":"4703d7bc-c574-4b48-8a41-00769bb48ad5","remote_addr":"[fd00:1122:3344:101::a]:41446","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25640,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-f0001f13-0a78-4e17-bdab-035b3367891b-cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.056410014Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f0001f13-0a78-4e17-bdab-035b3367891b-cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.059928049Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f0001f13-0a78-4e17-bdab-035b3367891b-cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.069908986Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0001f13-0a78-4e17-bdab-035b3367891b/.zfs/snapshot/cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.070101509Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.07015447Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.070200711Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.070238302Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.070286602Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.07392879Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.078114355Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"worker"}
{"msg":"region f0001f13-0a78-4e17-bdab-035b3367891b running snapshot cfc9d963-cb47-49ce-bf1c-ee0920dc9aff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.078123656Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"path is \"/data/regions/f0001f13-0a78-4e17-bdab-035b3367891b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.427329706Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"dataset is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.435985272Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"datafile"}
{"msg":"snapshot is oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b@cfc9d963-cb47-49ce-bf1c-ee0920dc9aff","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.445270798Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f2cfe38c-4b46-48f3-8ad4-e6d8ce866283/crucible/regions/f0001f13-0a78-4e17-bdab-035b3367891b\", \"cfc9d963-cb47-49ce-bf1c-ee0920dc9aff\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.445317349Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:15.453993666Z","hostname":"oxz_crucible_719c7f8b-413b-4d86-95bd-6f1e790a3318","pid":27206,"uri":"/crucible/0/regions/f0001f13-0a78-4e17-bdab-035b3367891b/snapshots/cfc9d963-cb47-49ce-bf1c-ee0920dc9aff/run","method":"POST","req_id":"47b68a1c-bf94-4d80-8180-b356f89f5204","remote_addr":"[fd00:1122:3344:101::a]:41446","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27283,"response_code":"200"}