[ Oct 18 05:35:47 Enabled. ]
[ Oct 18 05:35:47 Rereading configuration. ]
[ Oct 18 05:35:49 Rereading configuration. ]
[ Oct 18 05:35:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:54.365648859Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:54.367293232Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:54.367304242Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:54.38167Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:54.532617473Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.514939722Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62214"}
{"msg":"region 61e3e693-2344-46d4-b826-36bed4ea8831 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.546300328Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.549775088Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions","method":"POST","req_id":"11f1fa85-b498-423f-a8f8-8f525629a41d","remote_addr":"[fd00:1122:3344:101::a]:62214","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":11798,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.550960321Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.560612014Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.560642294Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"61e3e693-2344-46d4-b826-36bed4ea8831\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/61e3e693-2344-46d4-b826-36bed4ea8831\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.645624652Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"region":"61e3e693-2344-46d4-b826-36bed4ea8831","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"61e3e693-2344-46d4-b826-36bed4ea8831\" \"--data\" \"/data/regions/61e3e693-2344-46d4-b826-36bed4ea8831\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.64953995Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"region":"61e3e693-2344-46d4-b826-36bed4ea8831","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.784391394Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"region":"61e3e693-2344-46d4-b826-36bed4ea8831","component":"worker"}
{"msg":"region 61e3e693-2344-46d4-b826-36bed4ea8831 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.784452935Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.784523536Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.786788402Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.791736521Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.809382333Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.812518316Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.819182539Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.81925469Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.819306661Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.819345542Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.823167148Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.827346062Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.928288568Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions","method":"POST","req_id":"afe1ea32-b093-401c-bf8e-7cde3ea0767a","remote_addr":"[fd00:1122:3344:101::a]:62214","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:37.923547008Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47433"}
{"msg":"region 7f170d03-8449-4b11-ad68-f01a77e5ae4d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:37.92365134Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:37.924239532Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:37.924256742Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions","method":"POST","req_id":"0495ab46-16a9-410c-a6fa-2c5e4adb96ce","remote_addr":"[fd00:1122:3344:101::a]:47433","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":633,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/7f170d03-8449-4b11-ad68-f01a77e5ae4d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:37.935542637Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/7f170d03-8449-4b11-ad68-f01a77e5ae4d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:37.935570808Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7f170d03-8449-4b11-ad68-f01a77e5ae4d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7f170d03-8449-4b11-ad68-f01a77e5ae4d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.001319331Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"region":"7f170d03-8449-4b11-ad68-f01a77e5ae4d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7f170d03-8449-4b11-ad68-f01a77e5ae4d\" \"--data\" \"/data/regions/7f170d03-8449-4b11-ad68-f01a77e5ae4d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.001375902Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"region":"7f170d03-8449-4b11-ad68-f01a77e5ae4d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.074981083Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"region":"7f170d03-8449-4b11-ad68-f01a77e5ae4d","component":"worker"}
{"msg":"region 7f170d03-8449-4b11-ad68-f01a77e5ae4d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.075034794Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.075108105Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.081158416Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.081238528Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.081319939Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.08133152Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.08136954Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.081445742Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.08182438Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.081925631Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.081983253Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.082037504Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.089805769Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7f170d03-8449-4b11-ad68-f01a77e5ae4d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.09035514Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7f170d03-8449-4b11-ad68-f01a77e5ae4d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.093703697Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.101936911Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.102604995Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions","method":"POST","req_id":"e6cc94f8-4880-4761-bb53-10e73f478928","remote_addr":"[fd00:1122:3344:101::a]:47433","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":103,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7f170d03-8449-4b11-ad68-f01a77e5ae4d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.108823039Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7f170d03-8449-4b11-ad68-f01a77e5ae4d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.109245567Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.109330339Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.10938213Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.109434961Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.115033673Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:38.119812028Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.091915006Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions/61e3e693-2344-46d4-b826-36bed4ea8831","method":"GET","req_id":"8d0dde74-d812-4d3f-aeb6-3334d85c6cf2","remote_addr":"[fd00:1122:3344:101::a]:47433","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3174,"response_code":"200"}
{"msg":"path is \"/data/regions/61e3e693-2344-46d4-b826-36bed4ea8831\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.095381505Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"dataset is oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.104427336Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"snapshot is oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831@cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.113443056Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831\", \"cf4614e9-3156-42ca-be3e-103ee4c58112\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.113482427Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.126557348Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions/61e3e693-2344-46d4-b826-36bed4ea8831/snapshots/cf4614e9-3156-42ca-be3e-103ee4c58112","method":"GET","req_id":"4cce572a-c757-4580-9ed9-d3518cbdba40","remote_addr":"[fd00:1122:3344:101::a]:47433","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":31755,"response_code":"200"}
{"msg":"path is \"/data/regions/61e3e693-2344-46d4-b826-36bed4ea8831\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.129286972Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"dataset is oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.136926015Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"snapshot is oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831@cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.144579938Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831\", \"cf4614e9-3156-42ca-be3e-103ee4c58112\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.144607659Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 61e3e693-2344-46d4-b826-36bed4ea8831-cf4614e9-3156-42ca-be3e-103ee4c58112 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.153761312Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"applying SMF actions for region 61e3e693-2344-46d4-b826-36bed4ea8831 running snapshot cf4614e9-3156-42ca-be3e-103ee4c58112 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.153946105Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.153991096Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions/61e3e693-2344-46d4-b826-36bed4ea8831/snapshots/cf4614e9-3156-42ca-be3e-103ee4c58112/run","method":"POST","req_id":"90da4a2b-42a5-44cb-997b-2eff960c72e0","remote_addr":"[fd00:1122:3344:101::a]:47433","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24738,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.154566548Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.154650809Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.154734561Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.154764372Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.154772082Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.154818493Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.15518832Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.155248221Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.155293252Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.155337023Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.159273311Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-61e3e693-2344-46d4-b826-36bed4ea8831-cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.16072072Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-61e3e693-2344-46d4-b826-36bed4ea8831-cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.168388114Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-61e3e693-2344-46d4-b826-36bed4ea8831-cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.177580947Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/61e3e693-2344-46d4-b826-36bed4ea8831/.zfs/snapshot/cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.177874533Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.177936284Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.177978205Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.178017586Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.178068787Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.18173254Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.186121398Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"worker"}
{"msg":"region 61e3e693-2344-46d4-b826-36bed4ea8831 running snapshot cf4614e9-3156-42ca-be3e-103ee4c58112 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.186131368Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"path is \"/data/regions/61e3e693-2344-46d4-b826-36bed4ea8831\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.401829356Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"dataset is oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.410269715Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"datafile"}
{"msg":"snapshot is oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831@cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.418483149Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7f3257b7-3738-4198-968f-786ea4ecea64/crucible/regions/61e3e693-2344-46d4-b826-36bed4ea8831\", \"cf4614e9-3156-42ca-be3e-103ee4c58112\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.41851986Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.425946228Z","hostname":"oxz_crucible_72f99c19-02c9-4352-90fa-e64179b594eb","pid":21914,"uri":"/crucible/0/regions/61e3e693-2344-46d4-b826-36bed4ea8831/snapshots/cf4614e9-3156-42ca-be3e-103ee4c58112/run","method":"POST","req_id":"15bcef08-27b5-43ef-bf14-9664ed8f11a4","remote_addr":"[fd00:1122:3344:101::a]:47433","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24171,"response_code":"200"}