[ Aug 14 22:31:12 Enabled. ]
[ Aug 14 22:31:12 Rereading configuration. ]
[ Aug 14 22:31:15 Rereading configuration. ]
[ Aug 14 22:31:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.130451275Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.131613192Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.131626632Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.15135306Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:31:20.2773464Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.241978623Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64072"}
{"msg":"region 2f3b1407-1169-409e-9247-9f38737a5856 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.262781427Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.26848777Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.271966061Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"09038600-98a9-49db-9b76-f16df8ff4532","remote_addr":"[fd00:1122:3344:101::a]:64072","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15170,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.278598458Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.278618268Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2f3b1407-1169-409e-9247-9f38737a5856\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2f3b1407-1169-409e-9247-9f38737a5856\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.329043254Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"2f3b1407-1169-409e-9247-9f38737a5856","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2f3b1407-1169-409e-9247-9f38737a5856\" \"--data\" \"/data/regions/2f3b1407-1169-409e-9247-9f38737a5856\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.330904302Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"2f3b1407-1169-409e-9247-9f38737a5856","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.456177391Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"2f3b1407-1169-409e-9247-9f38737a5856","component":"worker"}
{"msg":"region 2f3b1407-1169-409e-9247-9f38737a5856 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.456232932Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.456332444Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.45679756Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.461123594Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.476497868Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.479686345Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.485129554Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.487240755Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.487326036Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.487389787Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.490988244Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.491370475Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"efaa8d54-0dc0-4925-ac15-ef0f7e0e2af9","remote_addr":"[fd00:1122:3344:101::a]:64072","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":101,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:32:37.495289333Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.559215365Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43581"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.559320666Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/2f3b1407-1169-409e-9247-9f38737a5856","method":"GET","req_id":"561c24b9-129f-4d1b-8fdc-197ccc21b44e","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"path is \"/data/regions/2f3b1407-1169-409e-9247-9f38737a5856\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.565639349Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.575586094Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"snapshot is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856@f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.584136159Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856\", \"f199dc8b-a47b-43bd-b61e-cf562c2a2ad7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.584167759Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.595235061Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/2f3b1407-1169-409e-9247-9f38737a5856/snapshots/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","method":"GET","req_id":"d9606b3f-1bde-402d-8f51-0ef8c270eee2","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":32505,"response_code":"200"}
{"msg":"path is \"/data/regions/2f3b1407-1169-409e-9247-9f38737a5856\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.598098213Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.605914997Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"snapshot is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856@f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.614042516Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856\", \"f199dc8b-a47b-43bd-b61e-cf562c2a2ad7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.614068806Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2f3b1407-1169-409e-9247-9f38737a5856-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.623400832Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"applying SMF actions for region 2f3b1407-1169-409e-9247-9f38737a5856 running snapshot f199dc8b-a47b-43bd-b61e-cf562c2a2ad7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.624067752Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.624099683Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/2f3b1407-1169-409e-9247-9f38737a5856/snapshots/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7/run","method":"POST","req_id":"5cd98f3f-a6f4-4b3c-9601-a7901751c790","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26034,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627510192Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627598474Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627680175Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627687965Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627731756Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627779896Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627874188Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.627944459Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.62799166Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.62804039Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.635167564Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-2f3b1407-1169-409e-9247-9f38737a5856-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.635622981Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2f3b1407-1169-409e-9247-9f38737a5856-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.638388201Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2f3b1407-1169-409e-9247-9f38737a5856-f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.647095419Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f3b1407-1169-409e-9247-9f38737a5856/.zfs/snapshot/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.647414983Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.647470244Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.647510445Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.647551145Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.647599586Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.651022316Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.655792706Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"region 2f3b1407-1169-409e-9247-9f38737a5856 running snapshot f199dc8b-a47b-43bd-b61e-cf562c2a2ad7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.655808266Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"path is \"/data/regions/2f3b1407-1169-409e-9247-9f38737a5856\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.939995056Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.948543111Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"snapshot is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856@f199dc8b-a47b-43bd-b61e-cf562c2a2ad7","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.957223398Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/2f3b1407-1169-409e-9247-9f38737a5856\", \"f199dc8b-a47b-43bd-b61e-cf562c2a2ad7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.957260698Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:03.965359927Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/2f3b1407-1169-409e-9247-9f38737a5856/snapshots/f199dc8b-a47b-43bd-b61e-cf562c2a2ad7/run","method":"POST","req_id":"8949e707-cdd0-446e-b8fd-e3d67cd6231e","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25403,"response_code":"200"}
{"msg":"region bf34655e-9207-4103-ba01-a08e0f192a54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.533894969Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.534062312Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"3d201631-15b2-44d3-be98-b90a205ed8e3","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":772,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.534091592Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.544791899Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.544819199Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bf34655e-9207-4103-ba01-a08e0f192a54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bf34655e-9207-4103-ba01-a08e0f192a54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.601417466Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"bf34655e-9207-4103-ba01-a08e0f192a54","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bf34655e-9207-4103-ba01-a08e0f192a54\" \"--data\" \"/data/regions/bf34655e-9207-4103-ba01-a08e0f192a54\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.601473436Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"bf34655e-9207-4103-ba01-a08e0f192a54","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.664118111Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"bf34655e-9207-4103-ba01-a08e0f192a54","component":"worker"}
{"msg":"region bf34655e-9207-4103-ba01-a08e0f192a54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.664170312Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.664261124Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.664838482Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.664916953Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.664998654Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665005135Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665047495Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665098656Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665482611Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665550472Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665603663Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.665651204Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.669248997Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.669683793Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.676373961Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.683596096Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.696053318Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.696350262Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.696413863Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.696462744Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.696507824Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.70028932Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.705965373Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:06.907010778Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"8f9cb023-43ae-466f-a3d8-bf0d60462ff0","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.785517647Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/bf34655e-9207-4103-ba01-a08e0f192a54","method":"GET","req_id":"555ee881-6c96-4cbb-a177-9f3ac7b6cc28","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/bf34655e-9207-4103-ba01-a08e0f192a54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.788215787Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/bf34655e-9207-4103-ba01-a08e0f192a54","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.79871263Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"region bf34655e-9207-4103-ba01-a08e0f192a54 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.806321811Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.806516294Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.806574945Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/bf34655e-9207-4103-ba01-a08e0f192a54","method":"DELETE","req_id":"d3a7743b-dc34-489f-b9bb-85c574ad3e47","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18396,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-bf34655e-9207-4103-ba01-a08e0f192a54 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.807771902Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.809498007Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/bf34655e-9207-4103-ba01-a08e0f192a54","method":"GET","req_id":"e7250f51-a21d-44ff-9e13-608193e6db11","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.811824441Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.811910002Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.811994173Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.812004394Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.812041504Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.812092225Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f3b1407-1169-409e-9247-9f38737a5856","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.81245805Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.812534251Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.812583842Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.812633043Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.816095643Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.817496734Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_2450844a-e7c2-49d0-8dc6-d59793fa3c37/crucible/regions/bf34655e-9207-4103-ba01-a08e0f192a54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.828242871Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"region":"bf34655e-9207-4103-ba01-a08e0f192a54","component":"worker"}
{"msg":"region bf34655e-9207-4103-ba01-a08e0f192a54 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.877661022Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-14T22:38:46.925908657Z","hostname":"oxz_crucible_bf4afffd-0248-4ed1-8c15-3ab55a5408c9","pid":24403,"uri":"/crucible/0/regions/bf34655e-9207-4103-ba01-a08e0f192a54","method":"GET","req_id":"83dfa63d-aaa3-4c56-b732-b5d65b3a2179","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":69,"response_code":"200"}