[ Oct  1 18:23:32 Enabled. ]
[ Oct  1 18:23:32 Rereading configuration. ]
[ Oct  1 18:23:33 Rereading configuration. ]
[ Oct  1 18:23:35 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:23:35.506712018Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:23:35.508395695Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:23:35.508406155Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:23:35.529684951Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:23:35.609742265Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.564744543Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53311"}
{"msg":"region bbcdf273-b537-4f3a-b5d8-89e63981dbed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.589324801Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.591748554Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"662b66ba-5412-4dc7-8b1e-19e080efa587","remote_addr":"[fd00:1122:3344:101::a]:53311","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":6728,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.592861509Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.602617553Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.602654204Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bbcdf273-b537-4f3a-b5d8-89e63981dbed\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.670633593Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"bbcdf273-b537-4f3a-b5d8-89e63981dbed","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bbcdf273-b537-4f3a-b5d8-89e63981dbed\" \"--data\" \"/data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.675167132Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"bbcdf273-b537-4f3a-b5d8-89e63981dbed","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.831216181Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"bbcdf273-b537-4f3a-b5d8-89e63981dbed","component":"worker"}
{"msg":"region bbcdf273-b537-4f3a-b5d8-89e63981dbed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.831275263Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.831363395Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.833860769Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.838391809Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.855532224Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.858406327Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.859187054Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.859287067Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.859347418Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.859400129Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.863378686Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.868345935Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:24:37.870993713Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"2dfcd538-53ad-43f5-ac9f-2fe85b63a21a","remote_addr":"[fd00:1122:3344:101::a]:53311","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1127,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.110481379Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61530"}
{"msg":"region 000c3aea-859f-4227-8a39-2d4bbecac67e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.110608861Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.111164503Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.111179734Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"9ee2a9ad-e104-4b05-9222-2ef54365a37c","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":572,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/000c3aea-859f-4227-8a39-2d4bbecac67e","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.121137952Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/000c3aea-859f-4227-8a39-2d4bbecac67e","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.121161653Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"000c3aea-859f-4227-8a39-2d4bbecac67e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/000c3aea-859f-4227-8a39-2d4bbecac67e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.190654215Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"000c3aea-859f-4227-8a39-2d4bbecac67e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"000c3aea-859f-4227-8a39-2d4bbecac67e\" \"--data\" \"/data/regions/000c3aea-859f-4227-8a39-2d4bbecac67e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.190708696Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"000c3aea-859f-4227-8a39-2d4bbecac67e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.265002664Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"000c3aea-859f-4227-8a39-2d4bbecac67e","component":"worker"}
{"msg":"region 000c3aea-859f-4227-8a39-2d4bbecac67e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.265044665Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.265125997Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-000c3aea-859f-4227-8a39-2d4bbecac67e","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.265517895Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-000c3aea-859f-4227-8a39-2d4bbecac67e","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.27030349Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.283272845Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.28716563Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"4216f41d-3ea4-48b0-9742-cfd0239a260e","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-000c3aea-859f-4227-8a39-2d4bbecac67e","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.287436136Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/000c3aea-859f-4227-8a39-2d4bbecac67e","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.287699211Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.287751792Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.287796144Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.287836725Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.291335781Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:45.298648411Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.782534043Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","method":"GET","req_id":"53c53d51-4ed6-46cb-99da-883af65510ac","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3091,"response_code":"200"}
{"msg":"path is \"/data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.786259525Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.795773004Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"snapshot is oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed@c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.80426458Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\", \"c69b7335-b7d0-4dc0-94a1-2792b4d75d33\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.804297511Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.817345836Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed/snapshots/c69b7335-b7d0-4dc0-94a1-2792b4d75d33","method":"GET","req_id":"c56dc9b7-eb04-478f-8cd9-befb95629b82","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":31090,"response_code":"200"}
{"msg":"path is \"/data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.821105429Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.829202496Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"snapshot is oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed@c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.836844644Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\", \"c69b7335-b7d0-4dc0-94a1-2792b4d75d33\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.836879304Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot bbcdf273-b537-4f3a-b5d8-89e63981dbed-c69b7335-b7d0-4dc0-94a1-2792b4d75d33 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.844618284Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions for region bbcdf273-b537-4f3a-b5d8-89e63981dbed running snapshot c69b7335-b7d0-4dc0-94a1-2792b4d75d33 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.844820728Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.844842399Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed/snapshots/c69b7335-b7d0-4dc0-94a1-2792b4d75d33/run","method":"POST","req_id":"c52ab0a7-ee91-4a1f-9cdd-da6b6d408fc4","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24257,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-bbcdf273-b537-4f3a-b5d8-89e63981dbed-c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.846998206Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-bbcdf273-b537-4f3a-b5d8-89e63981dbed-c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.851410583Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-bbcdf273-b537-4f3a-b5d8-89e63981dbed-c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.860703756Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed/.zfs/snapshot/c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.860904631Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.860954242Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.860994463Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.861032683Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.861087945Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.864737485Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.869651502Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"region bbcdf273-b537-4f3a-b5d8-89e63981dbed running snapshot c69b7335-b7d0-4dc0-94a1-2792b4d75d33 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:46.869665363Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"path is \"/data/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:47.050400252Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:47.058824627Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"snapshot is oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed@c69b7335-b7d0-4dc0-94a1-2792b4d75d33","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:47.066346401Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed\", \"c69b7335-b7d0-4dc0-94a1-2792b4d75d33\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:47.066373982Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:47.075214316Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions/bbcdf273-b537-4f3a-b5d8-89e63981dbed/snapshots/c69b7335-b7d0-4dc0-94a1-2792b4d75d33/run","method":"POST","req_id":"c2b415b4-7f44-47f6-8dd8-4cf6c25b3174","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24846,"response_code":"200"}
{"msg":"region 91ee6808-2ff3-46f7-81df-5223d3ab8541 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.573674098Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.573871842Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"4629f050-1e9e-4935-811c-0b1f7cbdf170","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":263,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.573917853Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/91ee6808-2ff3-46f7-81df-5223d3ab8541","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.58334719Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4562b4bc-fd1e-41e9-b26e-5a5229354600/crucible/regions/91ee6808-2ff3-46f7-81df-5223d3ab8541","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.58337782Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"91ee6808-2ff3-46f7-81df-5223d3ab8541\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/91ee6808-2ff3-46f7-81df-5223d3ab8541\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.646143796Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"91ee6808-2ff3-46f7-81df-5223d3ab8541","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"91ee6808-2ff3-46f7-81df-5223d3ab8541\" \"--data\" \"/data/regions/91ee6808-2ff3-46f7-81df-5223d3ab8541\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.646191797Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"91ee6808-2ff3-46f7-81df-5223d3ab8541","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.715190008Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"region":"91ee6808-2ff3-46f7-81df-5223d3ab8541","component":"worker"}
{"msg":"region 91ee6808-2ff3-46f7-81df-5223d3ab8541 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.7152404Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.715329991Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-91ee6808-2ff3-46f7-81df-5223d3ab8541","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.716678831Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-91ee6808-2ff3-46f7-81df-5223d3ab8541","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.719947953Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.726908215Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.729359189Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"ed9c9980-1214-43eb-9e86-b838a32f3f8a","remote_addr":"[fd00:1122:3344:101::a]:61530","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-91ee6808-2ff3-46f7-81df-5223d3ab8541","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.730460323Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/91ee6808-2ff3-46f7-81df-5223d3ab8541","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.730703298Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.73076185Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.73080347Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.730844722Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.734715676Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-01T18:29:50.745259787Z","hostname":"oxz_crucible_8b5786eb-64ec-4039-b083-e119adf32e41","pid":21860,"component":"worker"}