[ Aug  5 16:46:21 Enabled. ]
[ Aug  5 16:46:21 Rereading configuration. ]
[ Aug  5 16:46:22 Rereading configuration. ]
[ Aug  5 16:46:32 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:32.684642726Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:32.685506781Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:32.685517691Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:32.699473739Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:32.998968807Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.440536421Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:59021"}
{"msg":"region 833428f1-3860-4bd8-be35-33c39418f2cd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.448163931Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.451195982Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.451207903Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions","method":"POST","req_id":"fbe5d82c-b6f9-43f6-8c2a-a0b7b5d3b89b","remote_addr":"[fd00:1122:3344:101::c]:59021","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":5172,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible/regions/833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.459595816Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible/regions/833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.459662047Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"833428f1-3860-4bd8-be35-33c39418f2cd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/833428f1-3860-4bd8-be35-33c39418f2cd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.541033365Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"833428f1-3860-4bd8-be35-33c39418f2cd","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"833428f1-3860-4bd8-be35-33c39418f2cd\" \"--data\" \"/data/regions/833428f1-3860-4bd8-be35-33c39418f2cd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.542901236Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"833428f1-3860-4bd8-be35-33c39418f2cd","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.68566986Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"833428f1-3860-4bd8-be35-33c39418f2cd","component":"worker"}
{"msg":"region 833428f1-3860-4bd8-be35-33c39418f2cd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.685744632Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.685976596Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.686457934Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.692154321Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.706973174Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.713846461Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.714690286Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.714769417Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.714810137Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.714852118Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.727941781Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.735730394Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.744922291Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions","method":"POST","req_id":"c4ba4b16-a519-48b4-bfc8-d69e8343bc36","remote_addr":"[fd00:1122:3344:101::c]:59021","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.040612667Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions/833428f1-3860-4bd8-be35-33c39418f2cd","method":"GET","req_id":"c1d9625e-6da9-4848-81dd-efc3146b2480","remote_addr":"[fd00:1122:3344:101::c]:59021","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/833428f1-3860-4bd8-be35-33c39418f2cd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.043264612Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"dataset is oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible/regions/833428f1-3860-4bd8-be35-33c39418f2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.053397545Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"region 833428f1-3860-4bd8-be35-33c39418f2cd state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.0619207Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.062127184Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.062207985Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions/833428f1-3860-4bd8-be35-33c39418f2cd","method":"DELETE","req_id":"38d765bb-5d81-4de5-9272-9c8dfa05e14d","remote_addr":"[fd00:1122:3344:101::c]:59021","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18967,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-833428f1-3860-4bd8-be35-33c39418f2cd (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.063169131Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.065611263Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions/833428f1-3860-4bd8-be35-33c39418f2cd","method":"GET","req_id":"ef09818b-7d60-4cc0-ac2e-c24958c96bd5","remote_addr":"[fd00:1122:3344:101::c]:59021","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":517,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.07073145Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible/regions/833428f1-3860-4bd8-be35-33c39418f2cd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.080779182Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"833428f1-3860-4bd8-be35-33c39418f2cd","component":"worker"}
{"msg":"region 833428f1-3860-4bd8-be35-33c39418f2cd state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.146899759Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.180277708Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions/833428f1-3860-4bd8-be35-33c39418f2cd","method":"GET","req_id":"0e73073d-8f0c-43b9-8141-9f9b756db5a1","remote_addr":"[fd00:1122:3344:101::c]:59021","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.003887347Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35033"}
{"msg":"region 5ab8973c-38e7-4f44-a8e1-e33e897b90c9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.00402477Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.006824987Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.006850098Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions","method":"POST","req_id":"fc9bd567-781d-4841-a301-b735f9cf6720","remote_addr":"[fd00:1122:3344:101::a]:35033","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2895,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible/regions/5ab8973c-38e7-4f44-a8e1-e33e897b90c9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.017650472Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crucible/regions/5ab8973c-38e7-4f44-a8e1-e33e897b90c9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.017682632Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5ab8973c-38e7-4f44-a8e1-e33e897b90c9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5ab8973c-38e7-4f44-a8e1-e33e897b90c9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.067296358Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"5ab8973c-38e7-4f44-a8e1-e33e897b90c9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5ab8973c-38e7-4f44-a8e1-e33e897b90c9\" \"--data\" \"/data/regions/5ab8973c-38e7-4f44-a8e1-e33e897b90c9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.069188891Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"5ab8973c-38e7-4f44-a8e1-e33e897b90c9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.134341122Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"region":"5ab8973c-38e7-4f44-a8e1-e33e897b90c9","component":"worker"}
{"msg":"region 5ab8973c-38e7-4f44-a8e1-e33e897b90c9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.134372112Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.134494564Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5ab8973c-38e7-4f44-a8e1-e33e897b90c9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.135886568Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5ab8973c-38e7-4f44-a8e1-e33e897b90c9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.14124999Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.154655768Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5ab8973c-38e7-4f44-a8e1-e33e897b90c9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.157812572Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5ab8973c-38e7-4f44-a8e1-e33e897b90c9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.158126107Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.158186039Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.158230779Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.15827323Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.161964503Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.16651451Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:12.281810327Z","hostname":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","pid":29803,"uri":"/crucible/0/regions","method":"POST","req_id":"8ec3146f-e54a-4ccb-b9df-3a424f92a35e","remote_addr":"[fd00:1122:3344:101::a]:35033","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":611,"response_code":"200"}