[ Dec 12 22:34:12 Enabled. ]
[ Dec 12 22:34:12 Rereading configuration. ]
[ Dec 12 22:34:15 Rereading configuration. ]
[ Dec 12 22:34:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:18.943192924Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:18.947305774Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:18.947321685Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:18.964847989Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:19.0687537Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.389223646Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45582"}
{"msg":"region 88f141ca-cc1e-4592-8bdd-21bac9c7f477 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.406094358Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.408718029Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions","method":"POST","req_id":"29bfa9be-e9c9-425c-9b8a-42a0ae676e0d","remote_addr":"[fd00:1122:3344:101::c]:45582","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":4796,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.40873938Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.417873449Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.41793437Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"88f141ca-cc1e-4592-8bdd-21bac9c7f477\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.492316281Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"88f141ca-cc1e-4592-8bdd-21bac9c7f477","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"88f141ca-cc1e-4592-8bdd-21bac9c7f477\" \"--data\" \"/data/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.494406402Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"88f141ca-cc1e-4592-8bdd-21bac9c7f477","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.60990071Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"88f141ca-cc1e-4592-8bdd-21bac9c7f477","component":"worker"}
{"msg":"region 88f141ca-cc1e-4592-8bdd-21bac9c7f477 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.609991372Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.610215226Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.610622774Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.613907529Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.622930516Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.626813623Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.627582638Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.627651909Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.62769462Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.627737701Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.632090856Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.637230887Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:42.675391967Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions","method":"POST","req_id":"73feb212-9c96-4492-bd0f-0d668f6a000d","remote_addr":"[fd00:1122:3344:101::c]:45582","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.456789332Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","method":"GET","req_id":"3751d99b-244d-4d79-b3fd-b2f6c4bdd7a3","remote_addr":"[fd00:1122:3344:101::c]:45582","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.460604967Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"dataset is oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.471879199Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"region 88f141ca-cc1e-4592-8bdd-21bac9c7f477 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.481870775Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.48210116Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.482164481Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","method":"DELETE","req_id":"2f86e623-70d2-4b61-9846-0299a7c8d893","remote_addr":"[fd00:1122:3344:101::c]:45582","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21575,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.484953736Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","method":"GET","req_id":"b1902232-97c7-43d9-974a-4d607daf3745","remote_addr":"[fd00:1122:3344:101::c]:45582","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-88f141ca-cc1e-4592-8bdd-21bac9c7f477 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.485143329Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.489282951Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.5004688Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"88f141ca-cc1e-4592-8bdd-21bac9c7f477","component":"worker"}
{"msg":"region 88f141ca-cc1e-4592-8bdd-21bac9c7f477 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.561085041Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:34:43.609354999Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions/88f141ca-cc1e-4592-8bdd-21bac9c7f477","method":"GET","req_id":"d0b93686-fe40-420a-85f5-a0f0adcf719c","remote_addr":"[fd00:1122:3344:101::c]:45582","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.445546467Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59513"}
{"msg":"region 93074bf6-0e13-44ef-a787-a3578e290067 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.445734921Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.445911194Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions","method":"POST","req_id":"baa90258-fcef-4987-9501-c9bb7669f52e","remote_addr":"[fd00:1122:3344:101::a]:59513","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":215,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.445926975Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible/regions/93074bf6-0e13-44ef-a787-a3578e290067","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.455440311Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3c596d62-0729-49f7-ab54-437e3babc937/crucible/regions/93074bf6-0e13-44ef-a787-a3578e290067","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.455468752Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"93074bf6-0e13-44ef-a787-a3578e290067\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/93074bf6-0e13-44ef-a787-a3578e290067\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.506418623Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"93074bf6-0e13-44ef-a787-a3578e290067","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"93074bf6-0e13-44ef-a787-a3578e290067\" \"--data\" \"/data/regions/93074bf6-0e13-44ef-a787-a3578e290067\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.506491934Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"93074bf6-0e13-44ef-a787-a3578e290067","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.581356744Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"region":"93074bf6-0e13-44ef-a787-a3578e290067","component":"worker"}
{"msg":"region 93074bf6-0e13-44ef-a787-a3578e290067 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.581402256Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.581472467Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-93074bf6-0e13-44ef-a787-a3578e290067","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.582845154Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-93074bf6-0e13-44ef-a787-a3578e290067","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.585940555Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.592939162Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-93074bf6-0e13-44ef-a787-a3578e290067","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.596115585Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/93074bf6-0e13-44ef-a787-a3578e290067","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.59639386Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.596462231Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.596506692Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.596553443Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.600499111Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.606850165Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T22:35:27.651460051Z","hostname":"oxz_crucible_3b623f90-fc4f-4177-9f01-d4b6390873db","pid":24017,"uri":"/crucible/0/regions","method":"POST","req_id":"6c1a02ed-6b3f-44d2-881e-28653450cba0","remote_addr":"[fd00:1122:3344:101::a]:59513","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":"200"}