[ Aug 17 05:18:49 Enabled. ]
[ Aug 17 05:18:49 Rereading configuration. ]
[ Aug 17 05:18:51 Rereading configuration. ]
[ Aug 17 05:18:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:18:54.499333491Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:18:54.500235689Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:18:54.500246189Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:18:54.516740019Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:18:54.68661198Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.082507506Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56205"}
{"msg":"region 4821512a-25aa-4055-ad64-8d05c455841f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.105802233Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.110677725Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.114483427Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions","method":"POST","req_id":"0a960ec5-f42f-44ef-8fdd-6763fe241bcb","remote_addr":"[fd00:1122:3344:101::a]:56205","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":16581,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.119918653Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.119936193Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4821512a-25aa-4055-ad64-8d05c455841f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4821512a-25aa-4055-ad64-8d05c455841f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.16781697Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"4821512a-25aa-4055-ad64-8d05c455841f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4821512a-25aa-4055-ad64-8d05c455841f\" \"--data\" \"/data/regions/4821512a-25aa-4055-ad64-8d05c455841f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.169834647Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"4821512a-25aa-4055-ad64-8d05c455841f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.31400918Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"4821512a-25aa-4055-ad64-8d05c455841f","component":"worker"}
{"msg":"region 4821512a-25aa-4055-ad64-8d05c455841f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.314074351Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.314147661Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.314548495Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.320233113Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.34348754Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.346779888Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.354914657Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.356978015Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.357051925Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.357101406Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.361055799Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.368753164Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:20:09.441135789Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions","method":"POST","req_id":"2c1f1bf8-eed0-407f-b2ba-9430ed973860","remote_addr":"[fd00:1122:3344:101::a]:56205","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.568781281Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52827"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.568890631Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/4821512a-25aa-4055-ad64-8d05c455841f","method":"GET","req_id":"d1b8750e-e082-4dde-874a-f34163640970","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/4821512a-25aa-4055-ad64-8d05c455841f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.574439018Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"dataset is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.583227933Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"snapshot is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f@28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.591256231Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f\", \"28bb06e7-94ae-410d-bbac-c76370feda74\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.591282601Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.601905071Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/4821512a-25aa-4055-ad64-8d05c455841f/snapshots/28bb06e7-94ae-410d-bbac-c76370feda74","method":"GET","req_id":"a7cd6033-1883-4ff3-9b52-a6fb4e3caa8a","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":30451,"response_code":"200"}
{"msg":"path is \"/data/regions/4821512a-25aa-4055-ad64-8d05c455841f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.604930117Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"dataset is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.613616211Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"snapshot is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f@28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.621230095Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f\", \"28bb06e7-94ae-410d-bbac-c76370feda74\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.621259996Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 4821512a-25aa-4055-ad64-8d05c455841f-28bb06e7-94ae-410d-bbac-c76370feda74 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.629652217Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"applying SMF actions for region 4821512a-25aa-4055-ad64-8d05c455841f running snapshot 28bb06e7-94ae-410d-bbac-c76370feda74 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.630227262Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.630252162Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/4821512a-25aa-4055-ad64-8d05c455841f/snapshots/28bb06e7-94ae-410d-bbac-c76370feda74/run","method":"POST","req_id":"0e22bcd4-dc59-4c0e-8ff9-e7b7412fa97b","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25355,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633464989Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.63354856Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633633871Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633643721Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633686941Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633733642Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633828153Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633889883Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633938423Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.633988134Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.641026074Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-4821512a-25aa-4055-ad64-8d05c455841f-28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.641501327Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4821512a-25aa-4055-ad64-8d05c455841f-28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.644870486Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4821512a-25aa-4055-ad64-8d05c455841f-28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.654982482Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4821512a-25aa-4055-ad64-8d05c455841f/.zfs/snapshot/28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.655284175Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.655341825Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.655382195Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.655421046Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.655472066Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.659312599Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.663948608Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"region 4821512a-25aa-4055-ad64-8d05c455841f running snapshot 28bb06e7-94ae-410d-bbac-c76370feda74 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.663963138Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"path is \"/data/regions/4821512a-25aa-4055-ad64-8d05c455841f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.816816515Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"dataset is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.825279627Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"snapshot is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f@28bb06e7-94ae-410d-bbac-c76370feda74","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.833726998Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/4821512a-25aa-4055-ad64-8d05c455841f\", \"28bb06e7-94ae-410d-bbac-c76370feda74\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.833751159Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:07.841416734Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/4821512a-25aa-4055-ad64-8d05c455841f/snapshots/28bb06e7-94ae-410d-bbac-c76370feda74/run","method":"POST","req_id":"35f22a93-1429-4f7c-9aa7-924dea8c0d6d","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25121,"response_code":"200"}
{"msg":"region 9b940252-681b-4b0d-8d24-68e23daaf5d2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.140946598Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.14111317Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions","method":"POST","req_id":"9a85cce6-914f-490c-b952-f48e0dabc92b","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":228,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.14113823Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.150443529Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.150473589Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9b940252-681b-4b0d-8d24-68e23daaf5d2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.209163087Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"9b940252-681b-4b0d-8d24-68e23daaf5d2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9b940252-681b-4b0d-8d24-68e23daaf5d2\" \"--data\" \"/data/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.209222497Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"9b940252-681b-4b0d-8d24-68e23daaf5d2","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.273843266Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"9b940252-681b-4b0d-8d24-68e23daaf5d2","component":"worker"}
{"msg":"region 9b940252-681b-4b0d-8d24-68e23daaf5d2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.273895436Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.273984847Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.274622892Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.274643632Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.274727383Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.274734703Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.274777484Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.274829494Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.275202987Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.275267998Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.275317998Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.275365959Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.279749336Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.28018824Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.286512373Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.293055809Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.295812612Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.296092765Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.296152355Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.296196115Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.296234966Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.299614275Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.30496793Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:11.377538756Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions","method":"POST","req_id":"e3608e92-69af-4f5e-98f4-e76c306be4e2","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.665627906Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","method":"GET","req_id":"f5bf2db8-f906-471c-826b-dd6fdaf53b67","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"path is \"/data/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.668214128Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"dataset is oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.678471865Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"region 9b940252-681b-4b0d-8d24-68e23daaf5d2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.687103229Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.687358891Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.687424691Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","method":"DELETE","req_id":"9dfabbd1-1a2c-4003-b2fc-36b19cec5031","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19250,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-9b940252-681b-4b0d-8d24-68e23daaf5d2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.689020685Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.691044852Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","method":"GET","req_id":"231e74bc-bf66-4a53-b00b-ae61fb241772","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":567,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693256411Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693348231Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693430252Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693441072Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693479663Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693525923Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4821512a-25aa-4055-ad64-8d05c455841f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693843956Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693920586Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.693975207Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.694022217Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.697653298Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.699016399Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_45c22b91-71fc-4412-a2b9-449af41d4f0b/crucible/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.708183727Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"region":"9b940252-681b-4b0d-8d24-68e23daaf5d2","component":"worker"}
{"msg":"region 9b940252-681b-4b0d-8d24-68e23daaf5d2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.752900337Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-17T05:25:51.765162941Z","hostname":"oxz_crucible_35229b2c-ef6d-452d-b296-63e4a52b71c3","pid":19959,"uri":"/crucible/0/regions/9b940252-681b-4b0d-8d24-68e23daaf5d2","method":"GET","req_id":"9ea6b6e2-b16d-4d0f-9430-ebbcf4ec277b","remote_addr":"[fd00:1122:3344:101::a]:52827","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":"200"}