[ Jul 22 23:07:15 Enabled. ]
[ Jul 22 23:07:15 Rereading configuration. ]
[ Jul 22 23:07:18 Rereading configuration. ]
[ Jul 22 23:07:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:07:22.798215575Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:07:22.801822259Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:07:22.801834599Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:07:22.816880676Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:07:22.918297735Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.401773784Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46894"}
{"msg":"region 511c670d-afe5-4745-86de-0cbbc680ae3b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.420369445Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.427127009Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions","method":"POST","req_id":"e9f637b4-3aa0-4899-be88-401f4eda6451","remote_addr":"[fd00:1122:3344:101::a]:46894","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":14650,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.427147689Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.440647727Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.440679767Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"511c670d-afe5-4745-86de-0cbbc680ae3b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.496882432Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"511c670d-afe5-4745-86de-0cbbc680ae3b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"511c670d-afe5-4745-86de-0cbbc680ae3b\" \"--data\" \"/data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.499045936Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"511c670d-afe5-4745-86de-0cbbc680ae3b","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.654102891Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"511c670d-afe5-4745-86de-0cbbc680ae3b","component":"worker"}
{"msg":"region 511c670d-afe5-4745-86de-0cbbc680ae3b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.654159642Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.654240292Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.654736205Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.661495519Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.67851585Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.681545529Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.682240134Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.686441431Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.686500431Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.686542302Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.692947747Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.699766898Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:08:34.761268747Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions","method":"POST","req_id":"888fa749-7935-4ac2-8942-6ee2915b7428","remote_addr":"[fd00:1122:3344:101::a]:46894","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.702201071Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46552"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.705325901Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","method":"GET","req_id":"41a46713-dd17-49de-81fd-03639878dd01","remote_addr":"[fd00:1122:3344:101::a]:46552","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3097,"response_code":"200"}
{"msg":"path is \"/data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.707392115Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"dataset is oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.716373323Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"snapshot is oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b@9a8a0026-eb67-4908-babd-b4c60882cf96","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.724872228Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b\", \"9a8a0026-eb67-4908-babd-b4c60882cf96\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.724898468Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.736865806Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/511c670d-afe5-4745-86de-0cbbc680ae3b/snapshots/9a8a0026-eb67-4908-babd-b4c60882cf96","method":"GET","req_id":"d8087a75-7ade-484d-873a-b3afb75a6bf6","remote_addr":"[fd00:1122:3344:101::a]:46552","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":31369,"response_code":"200"}
{"msg":"path is \"/data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.737050347Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"dataset is oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.744684047Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"snapshot is oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b@9a8a0026-eb67-4908-babd-b4c60882cf96","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.75280063Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/511c670d-afe5-4745-86de-0cbbc680ae3b\", \"9a8a0026-eb67-4908-babd-b4c60882cf96\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.75282671Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 511c670d-afe5-4745-86de-0cbbc680ae3b-9a8a0026-eb67-4908-babd-b4c60882cf96 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.762488043Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"applying SMF actions for region 511c670d-afe5-4745-86de-0cbbc680ae3b running snapshot 9a8a0026-eb67-4908-babd-b4c60882cf96 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.763025696Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.763060496Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/511c670d-afe5-4745-86de-0cbbc680ae3b/snapshots/9a8a0026-eb67-4908-babd-b4c60882cf96/run","method":"POST","req_id":"11d113ed-3464-4ea8-b2c4-262c568259c1","remote_addr":"[fd00:1122:3344:101::a]:46552","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26037,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771161079Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771248969Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.77133819Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.77134806Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771404521Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771456781Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771785513Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771853063Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771901914Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.771955084Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.787743676Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-511c670d-afe5-4745-86de-0cbbc680ae3b-9a8a0026-eb67-4908-babd-b4c60882cf96","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.788203479Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-511c670d-afe5-4745-86de-0cbbc680ae3b-9a8a0026-eb67-4908-babd-b4c60882cf96","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.798803568Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-511c670d-afe5-4745-86de-0cbbc680ae3b-9a8a0026-eb67-4908-babd-b4c60882cf96","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.821907188Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b/.zfs/snapshot/9a8a0026-eb67-4908-babd-b4c60882cf96","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.82222248Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.822301181Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.822354951Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.822409052Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.822481972Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.829398614Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.836413312Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"region 511c670d-afe5-4745-86de-0cbbc680ae3b running snapshot 9a8a0026-eb67-4908-babd-b4c60882cf96 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:42.836428243Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:45.934793021Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37295"}
{"msg":"region cdacc612-608e-4145-8519-d8a0eed1faa6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:45.934896351Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:45.937097066Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:45.937111526Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions","method":"POST","req_id":"6543c7a9-a4d8-42c1-a783-e50c315771d4","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2237,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:45.947875906Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:45.947903756Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cdacc612-608e-4145-8519-d8a0eed1faa6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cdacc612-608e-4145-8519-d8a0eed1faa6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.044435232Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"cdacc612-608e-4145-8519-d8a0eed1faa6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cdacc612-608e-4145-8519-d8a0eed1faa6\" \"--data\" \"/data/regions/cdacc612-608e-4145-8519-d8a0eed1faa6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.044489853Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"cdacc612-608e-4145-8519-d8a0eed1faa6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.147789573Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions","method":"POST","req_id":"7fb856ad-43c9-4a85-89ef-6427bf235b8c","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":93,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.223326524Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"cdacc612-608e-4145-8519-d8a0eed1faa6","component":"worker"}
{"msg":"region cdacc612-608e-4145-8519-d8a0eed1faa6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.223368064Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.223507365Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224068738Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224151119Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224230929Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224237699Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.22428119Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.22433358Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224712713Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224774013Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224832943Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.224890564Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.233674631Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.234314925Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.241499622Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.264200479Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.273973422Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.274251314Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.274320295Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.274363075Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.274402995Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.281027388Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.291732537Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:13:46.83138082Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions","method":"POST","req_id":"49779198-1691-4a73-b370-7a7bf0f682ec","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:10.992440849Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","method":"GET","req_id":"14dabd1e-3922-44d2-b748-31939710d001","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"path is \"/data/regions/cdacc612-608e-4145-8519-d8a0eed1faa6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:10.995201197Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"dataset is oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.008345482Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"region cdacc612-608e-4145-8519-d8a0eed1faa6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.017973025Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.018207566Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.018279177Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","method":"DELETE","req_id":"c51645b8-8462-40be-af92-db767ceca41c","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23724,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-cdacc612-608e-4145-8519-d8a0eed1faa6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.019506885Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.020929544Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","method":"GET","req_id":"1310c542-dcf6-4d82-86ec-ccb87e93af5c","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.02492002Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.02500655Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025087901Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025099641Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025141381Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025203602Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/511c670d-afe5-4745-86de-0cbbc680ae3b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025542234Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025607204Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025657195Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.025710035Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.030638427Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.032143357Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_166fc5d5-ed26-44e3-acae-034d165b26da/crucible/regions/cdacc612-608e-4145-8519-d8a0eed1faa6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.041977681Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"region":"cdacc612-608e-4145-8519-d8a0eed1faa6","component":"worker"}
{"msg":"region cdacc612-608e-4145-8519-d8a0eed1faa6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.089513439Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-22T23:14:11.137622271Z","hostname":"oxz_crucible_30bb221a-dd8e-47c6-ac11-f1181d055778","pid":23556,"uri":"/crucible/0/regions/cdacc612-608e-4145-8519-d8a0eed1faa6","method":"GET","req_id":"e10f503f-3f8e-4803-bb8d-45e6b4ca57a9","remote_addr":"[fd00:1122:3344:101::a]:37295","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":71,"response_code":"200"}