[ Dec 18 22:13:58 Enabled. ]
[ Dec 18 22:13:58 Rereading configuration. ]
[ Dec 18 22:14:01 Rereading configuration. ]
[ Dec 18 22:14:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.83421533Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.839510134Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.839521714Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.854456267Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:05.92750994Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.943114161Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62740"}
{"msg":"region 3a963b0e-f837-4f47-bb70-e2dc43eb42fb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.965931669Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.968351356Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.972883945Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions","method":"POST","req_id":"f6533c8b-76e7-4019-a943-9481bb01d5df","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":8906,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.979091337Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:50.979117328Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3a963b0e-f837-4f47-bb70-e2dc43eb42fb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.048332166Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"3a963b0e-f837-4f47-bb70-e2dc43eb42fb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3a963b0e-f837-4f47-bb70-e2dc43eb42fb\" \"--data\" \"/data/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.05061783Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"3a963b0e-f837-4f47-bb70-e2dc43eb42fb","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.187032607Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"3a963b0e-f837-4f47-bb70-e2dc43eb42fb","component":"worker"}
{"msg":"region 3a963b0e-f837-4f47-bb70-e2dc43eb42fb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.187082348Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.187165739Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.187622758Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.192033945Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.204394888Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.207407967Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.208124691Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.208183892Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.208226333Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.208266293Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.212031497Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.216144698Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:51.22747709Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions","method":"POST","req_id":"43bdedf5-37d8-4c10-a35d-cf75d61a2bd4","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.333211035Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","method":"GET","req_id":"1c9ad9c5-2303-4aa0-9d0e-b648c4410fec","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"path is \"/data/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.33704616Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"dataset is oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.348007915Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"region 3a963b0e-f837-4f47-bb70-e2dc43eb42fb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.358820387Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.35898908Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.359053471Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","method":"DELETE","req_id":"3649ff7b-64e0-4a8e-95c4-e52130237cfc","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22039,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-3a963b0e-f837-4f47-bb70-e2dc43eb42fb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.360287266Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.362002739Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","method":"GET","req_id":"69fd67fc-a89e-4c18-8135-1e6bfdf599c3","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":41,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.365046799Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.375031085Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"3a963b0e-f837-4f47-bb70-e2dc43eb42fb","component":"worker"}
{"msg":"region 3a963b0e-f837-4f47-bb70-e2dc43eb42fb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.450189239Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:52.467921817Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions/3a963b0e-f837-4f47-bb70-e2dc43eb42fb","method":"GET","req_id":"162db215-9efe-4d0b-a24f-6cb5cb24d766","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":95,"response_code":"200"}
{"msg":"region f5fcc036-4529-42e9-836e-041d1accce6d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.289025837Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.289196071Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions","method":"POST","req_id":"94dc0e9e-3552-45a1-b336-ff56f360ff05","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":244,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.289216451Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible/regions/f5fcc036-4529-42e9-836e-041d1accce6d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.298095205Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7527a580-440a-4d33-b9e5-9f4ac1126696/crucible/regions/f5fcc036-4529-42e9-836e-041d1accce6d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.298151576Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f5fcc036-4529-42e9-836e-041d1accce6d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f5fcc036-4529-42e9-836e-041d1accce6d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.354290738Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"f5fcc036-4529-42e9-836e-041d1accce6d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f5fcc036-4529-42e9-836e-041d1accce6d\" \"--data\" \"/data/regions/f5fcc036-4529-42e9-836e-041d1accce6d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.356519481Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"f5fcc036-4529-42e9-836e-041d1accce6d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.427402162Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"region":"f5fcc036-4529-42e9-836e-041d1accce6d","component":"worker"}
{"msg":"region f5fcc036-4529-42e9-836e-041d1accce6d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.427442183Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.427538135Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f5fcc036-4529-42e9-836e-041d1accce6d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.428960543Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f5fcc036-4529-42e9-836e-041d1accce6d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.432304088Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.440116912Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f5fcc036-4529-42e9-836e-041d1accce6d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.443190262Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f5fcc036-4529-42e9-836e-041d1accce6d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.443484628Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.443537059Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.44357818Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.44361713Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.447925694Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.454634346Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-18T22:14:54.576250492Z","hostname":"oxz_crucible_218e3da3-c056-4f46-bedf-ff540f2a1d48","pid":23770,"uri":"/crucible/0/regions","method":"POST","req_id":"e04bcb70-aee8-4c35-b4e8-5e72a0ab5934","remote_addr":"[fd00:1122:3344:101::a]:62740","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":91,"response_code":"200"}