[ Oct 30 17:49:03 Enabled. ]
[ Oct 30 17:49:03 Rereading configuration. ]
[ Oct 30 17:49:04 Rereading configuration. ]
[ Oct 30 17:49:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8b4477e5-c445-4846-a713-78ddac5696a7/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8b4477e5-c445-4846-a713-78ddac5696a7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.610958992Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.612981173Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.612993873Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.644739663Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.809994437Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.385044139Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36193"}
{"msg":"region e64ae311-355a-45df-b1f7-e02a292657bd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.415829115Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.419779643Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions","method":"POST","req_id":"4b5303e1-17c2-42e0-a098-db57e8822488","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":11551,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.42085025Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_8b4477e5-c445-4846-a713-78ddac5696a7/crucible/regions/e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.429468712Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8b4477e5-c445-4846-a713-78ddac5696a7/crucible/regions/e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.429499273Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e64ae311-355a-45df-b1f7-e02a292657bd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e64ae311-355a-45df-b1f7-e02a292657bd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.489860582Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"region":"e64ae311-355a-45df-b1f7-e02a292657bd","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e64ae311-355a-45df-b1f7-e02a292657bd\" \"--data\" \"/data/regions/e64ae311-355a-45df-b1f7-e02a292657bd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.493612223Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"region":"e64ae311-355a-45df-b1f7-e02a292657bd","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.699922731Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions","method":"POST","req_id":"ca813b41-bd27-4bbb-9b2e-b4e14f51e436","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.747168462Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"region":"e64ae311-355a-45df-b1f7-e02a292657bd","component":"worker"}
{"msg":"region e64ae311-355a-45df-b1f7-e02a292657bd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.747235345Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.747419371Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.749742882Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.755791164Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.772215677Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.777875935Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.784749966Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.784819578Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.784858219Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.784905221Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.789829613Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.794400123Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:09.365192136Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions","method":"POST","req_id":"8f2c34a6-8c72-4e6a-a950-a25e8253f0f4","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.029056483Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions/e64ae311-355a-45df-b1f7-e02a292657bd","method":"GET","req_id":"971cc843-f195-4043-8080-91d7d53a6335","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"path is \"/data/regions/e64ae311-355a-45df-b1f7-e02a292657bd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.032862996Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"dataset is oxp_8b4477e5-c445-4846-a713-78ddac5696a7/crucible/regions/e64ae311-355a-45df-b1f7-e02a292657bd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.04531426Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"region e64ae311-355a-45df-b1f7-e02a292657bd state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.055685803Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.055900481Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.055994034Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions/e64ae311-355a-45df-b1f7-e02a292657bd","method":"DELETE","req_id":"c437105c-7d0f-49a8-903b-14f49274bf2b","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23162,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e64ae311-355a-45df-b1f7-e02a292657bd (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.057002199Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.060099377Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions/e64ae311-355a-45df-b1f7-e02a292657bd","method":"GET","req_id":"0886c88f-8b29-4983-bc8e-c27e228ba6c1","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":37,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.061627341Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8b4477e5-c445-4846-a713-78ddac5696a7/crucible/regions/e64ae311-355a-45df-b1f7-e02a292657bd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.072912115Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"region":"e64ae311-355a-45df-b1f7-e02a292657bd","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.150979963Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions/e64ae311-355a-45df-b1f7-e02a292657bd","method":"GET","req_id":"b9faf5b4-aea6-4139-8640-631583ae9db9","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"region e64ae311-355a-45df-b1f7-e02a292657bd state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.15577164Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.312212686Z","hostname":"oxz_crucible_0be13225-a78b-4173-9c0a-759ebd323d2d","pid":20871,"uri":"/crucible/0/regions/e64ae311-355a-45df-b1f7-e02a292657bd","method":"GET","req_id":"f8a4fc51-969d-49b1-a279-c6295874a1c1","remote_addr":"[fd00:1122:3344:101::a]:36193","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":51,"response_code":"200"}