[ Aug 13 21:52:33 Enabled. ]
[ Aug 13 21:52:33 Rereading configuration. ]
[ Aug 13 21:52:34 Rereading configuration. ]
[ Aug 13 21:52:35 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3813b565-0c80-4aa9-986b-1dfd99c3d209/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_3813b565-0c80-4aa9-986b-1dfd99c3d209/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:52:35.682900621Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:52:35.684058893Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:52:35.684070293Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:52:35.707371383Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:52:35.854295024Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.558416158Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53934"}
{"msg":"region 8d9f7f78-9a86-450d-b985-1f1c019dda19 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.585283716Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.594819567Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions","method":"POST","req_id":"78a09249-6f59-44b2-8db7-6ffd6c9aa23b","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19030,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.595891567Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3813b565-0c80-4aa9-986b-1dfd99c3d209/crucible/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.60951557Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3813b565-0c80-4aa9-986b-1dfd99c3d209/crucible/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.609545661Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8d9f7f78-9a86-450d-b985-1f1c019dda19\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.673218928Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"region":"8d9f7f78-9a86-450d-b985-1f1c019dda19","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8d9f7f78-9a86-450d-b985-1f1c019dda19\" \"--data\" \"/data/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.675136691Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"region":"8d9f7f78-9a86-450d-b985-1f1c019dda19","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:10.891032695Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions","method":"POST","req_id":"1fd2bc2a-71e7-4009-be6b-3f92e05749b5","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.024588669Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"region":"8d9f7f78-9a86-450d-b985-1f1c019dda19","component":"worker"}
{"msg":"region 8d9f7f78-9a86-450d-b985-1f1c019dda19 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.024632391Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.024737834Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.025143835Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.033410201Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.051088417Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.057334458Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.069228914Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.071479536Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.071535267Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.071575009Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.078948171Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.085787529Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:11.181041162Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions","method":"POST","req_id":"6163f8d3-78c9-42bb-ad45-fdccdb57242a","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":646,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.709107672Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","method":"GET","req_id":"889052c3-bb59-428e-864c-e15e97a7bb8d","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"path is \"/data/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.716462244Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"dataset is oxp_3813b565-0c80-4aa9-986b-1dfd99c3d209/crucible/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.72616142Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"region 8d9f7f78-9a86-450d-b985-1f1c019dda19 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.73525094Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.735459726Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.735524477Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","method":"DELETE","req_id":"24e664c1-374c-4c59-b962-0249f33cd41f","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19925,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8d9f7f78-9a86-450d-b985-1f1c019dda19 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.736663399Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.739150857Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","method":"GET","req_id":"87949cb1-0d24-40f7-983b-5792991baa70","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.743867396Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3813b565-0c80-4aa9-986b-1dfd99c3d209/crucible/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.754273592Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"region":"8d9f7f78-9a86-450d-b985-1f1c019dda19","component":"worker"}
{"msg":"region 8d9f7f78-9a86-450d-b985-1f1c019dda19 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.811191624Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-13T21:59:35.82562168Z","hostname":"oxz_crucible_10106c78-85bb-4f37-b254-24b8489a754a","pid":20680,"uri":"/crucible/0/regions/8d9f7f78-9a86-450d-b985-1f1c019dda19","method":"GET","req_id":"01232f0b-064a-4d9e-859c-dd0c83e62680","remote_addr":"[fd00:1122:3344:101::a]:53934","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":58,"response_code":"200"}