[ Mar  7 22:47:01 Enabled. ]
[ Mar  7 22:47:01 Rereading configuration. ]
[ Mar  7 22:47:05 Rereading configuration. ]
[ Mar  7 22:47:10 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:10.066833028Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:10.069708964Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:10.069727105Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:10.436790091Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.262453139Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:62676"}
{"msg":"region dfe1592e-0577-4ebb-a5af-f8b0449b679a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.323273187Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.323623017Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"922b7f59-6614-4cb3-a748-e214b123e1e4","remote_addr":"[fd00:1122:3344:101::b]:62676","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":1525,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.324316998Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.334095883Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.334169705Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dfe1592e-0577-4ebb-a5af-f8b0449b679a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.409000886Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"dfe1592e-0577-4ebb-a5af-f8b0449b679a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.497965303Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"0ed1cc1b-51c8-46d7-ada9-70bbf4bd903a","remote_addr":"[fd00:1122:3344:101::b]:62676","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.543803676Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"dfe1592e-0577-4ebb-a5af-f8b0449b679a","component":"worker"}
{"msg":"region dfe1592e-0577-4ebb-a5af-f8b0449b679a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.543827097Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.544108766Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.544631962Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.551651944Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.57040064Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.577197235Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.578042031Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.578133484Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.578183815Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.578219406Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.584845496Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.591992022Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.072795114Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"1669cb62-3460-4d75-8a5e-62e8e9b9f7d9","remote_addr":"[fd00:1122:3344:101::b]:62676","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.879914071Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:51664"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.880032454Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","method":"GET","req_id":"847bf397-ba42-400a-b9eb-ab4658c4097c","remote_addr":"[fd00:1122:3344:101::b]:51664","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.880746156Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.889949814Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"region dfe1592e-0577-4ebb-a5af-f8b0449b679a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.899059429Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.899289846Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.899349008Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","method":"DELETE","req_id":"077c8d5a-62b5-469a-8b26-d3e12de0b652","remote_addr":"[fd00:1122:3344:101::b]:51664","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19130,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.899503953Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","method":"GET","req_id":"cd3f4aae-d4da-48fc-b3ba-daa841e8ec7a","remote_addr":"[fd00:1122:3344:101::b]:51664","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-dfe1592e-0577-4ebb-a5af-f8b0449b679a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.900586715Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.906211315Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.915935809Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"dfe1592e-0577-4ebb-a5af-f8b0449b679a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.960844835Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","method":"GET","req_id":"6f77a9de-3732-4e6a-965c-3551304fe039","remote_addr":"[fd00:1122:3344:101::b]:51664","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region dfe1592e-0577-4ebb-a5af-f8b0449b679a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.98881152Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:32.044542393Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/dfe1592e-0577-4ebb-a5af-f8b0449b679a","method":"GET","req_id":"c2492dfc-724d-4d41-8150-7cb9544d0c9c","remote_addr":"[fd00:1122:3344:101::b]:51664","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":495,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.31571638Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61868"}
{"msg":"region 735c85e2-b0a4-48ab-beca-781fdb031882 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.345598943Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.345766488Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"ce1a70f5-796e-4992-8fa0-127c8d384a3a","remote_addr":"[fd00:1122:3344:101::a]:61868","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":210,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.345787568Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.354847892Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.354885093Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"735c85e2-b0a4-48ab-beca-781fdb031882\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/735c85e2-b0a4-48ab-beca-781fdb031882\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.400364377Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"735c85e2-b0a4-48ab-beca-781fdb031882","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.471728082Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"735c85e2-b0a4-48ab-beca-781fdb031882","component":"worker"}
{"msg":"region 735c85e2-b0a4-48ab-beca-781fdb031882 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.471774193Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.471886137Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.473342631Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.476461905Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.483612801Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.486494898Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.486737295Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.486793847Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.486835678Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.486874119Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.490361945Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.495066907Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:48:22.670584287Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"862d9551-7e4f-40b8-9935-0198ad5843e7","remote_addr":"[fd00:1122:3344:101::a]:61868","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.146351418Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63106"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.146510623Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/735c85e2-b0a4-48ab-beca-781fdb031882","method":"GET","req_id":"2596c697-a6b6-4aba-a9ca-67eadd0a0c42","remote_addr":"[fd00:1122:3344:101::a]:63106","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/735c85e2-b0a4-48ab-beca-781fdb031882\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.146632776Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.155801413Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"snapshot is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882@8358d21e-eb55-4006-a757-e89a24336027","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.164213547Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882\", \"8358d21e-eb55-4006-a757-e89a24336027\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.164250089Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.174722015Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/735c85e2-b0a4-48ab-beca-781fdb031882/snapshots/8358d21e-eb55-4006-a757-e89a24336027","method":"GET","req_id":"fb5350ad-b5db-4670-9a61-6dfa8e25a422","remote_addr":"[fd00:1122:3344:101::a]:63106","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28127,"response_code":"200"}
{"msg":"path is \"/data/regions/735c85e2-b0a4-48ab-beca-781fdb031882\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.174864349Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.182596412Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"snapshot is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882@8358d21e-eb55-4006-a757-e89a24336027","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.191196442Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/735c85e2-b0a4-48ab-beca-781fdb031882\", \"8358d21e-eb55-4006-a757-e89a24336027\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.191229413Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 735c85e2-b0a4-48ab-beca-781fdb031882-8358d21e-eb55-4006-a757-e89a24336027 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.203443722Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"applying SMF actions for region 735c85e2-b0a4-48ab-beca-781fdb031882 running snapshot 8358d21e-eb55-4006-a757-e89a24336027 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.204114142Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.204157134Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/735c85e2-b0a4-48ab-beca-781fdb031882/snapshots/8358d21e-eb55-4006-a757-e89a24336027/run","method":"POST","req_id":"dce35915-a45f-43fb-b36b-a3a9b1b31597","remote_addr":"[fd00:1122:3344:101::a]:63106","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29330,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-735c85e2-b0a4-48ab-beca-781fdb031882-8358d21e-eb55-4006-a757-e89a24336027","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.212808285Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-735c85e2-b0a4-48ab-beca-781fdb031882-8358d21e-eb55-4006-a757-e89a24336027","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.220469796Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-735c85e2-b0a4-48ab-beca-781fdb031882-8358d21e-eb55-4006-a757-e89a24336027","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.242937385Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/735c85e2-b0a4-48ab-beca-781fdb031882/.zfs/snapshot/8358d21e-eb55-4006-a757-e89a24336027","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.243264385Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.243325217Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.243377728Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.24342843Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.243494982Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.25138975Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.257827455Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"region 735c85e2-b0a4-48ab-beca-781fdb031882 running snapshot 8358d21e-eb55-4006-a757-e89a24336027 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:30.257839715Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.173848765Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45173"}
{"msg":"region b8b0b285-0c09-4a49-bee5-cd4b4cf327cc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.234052743Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.234206488Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"856742b9-e8b0-4286-8b5c-dbce96d6fde2","remote_addr":"[fd00:1122:3344:101::a]:45173","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":958,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.234227768Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.243708025Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.243749846Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b8b0b285-0c09-4a49-bee5-cd4b4cf327cc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.32698863Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.447607333Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","component":"worker"}
{"msg":"region b8b0b285-0c09-4a49-bee5-cd4b4cf327cc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.447678065Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.447773858Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.449648495Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.455427529Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.469619788Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.479651881Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.479888538Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.47995752Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.480013192Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.480077244Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.487213989Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.492439027Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions","method":"POST","req_id":"569c0b29-4687-4bd5-95f6-1f282a224212","remote_addr":"[fd00:1122:3344:101::a]:45173","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:53:33.500185291Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.579657689Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48526"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.579770002Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","method":"GET","req_id":"ca252ef4-30a1-49c2-9d95-5f9876fbd3ab","remote_addr":"[fd00:1122:3344:101::a]:48526","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"path is \"/data/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.579952288Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.591216258Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"region b8b0b285-0c09-4a49-bee5-cd4b4cf327cc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.599603652Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.599830538Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.599865869Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","method":"DELETE","req_id":"bfb61839-0bc7-4390-a749-0d7fc723d15c","remote_addr":"[fd00:1122:3344:101::a]:48526","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19941,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.600579981Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","method":"GET","req_id":"981bfeac-fec3-42f3-81dd-4e19ea3481a1","remote_addr":"[fd00:1122:3344:101::a]:48526","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":573,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-b8b0b285-0c09-4a49-bee5-cd4b4cf327cc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.601629923Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.609585033Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.619162822Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"region":"b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","component":"worker"}
{"msg":"region b8b0b285-0c09-4a49-bee5-cd4b4cf327cc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.665180642Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:54:07.729878696Z","hostname":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","pid":28728,"uri":"/crucible/0/regions/b8b0b285-0c09-4a49-bee5-cd4b4cf327cc","method":"GET","req_id":"d38baf64-b09d-43b5-a490-79d03dc3a211","remote_addr":"[fd00:1122:3344:101::a]:48526","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":55,"response_code":"200"}