[ Aug 19 19:03:09 Enabled. ]
[ Aug 19 19:03:09 Rereading configuration. ]
[ Aug 19 19:03:10 Rereading configuration. ]
[ Aug 19 19:03:12 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6a3ed70a-94ff-4809-9d6e-4532771366ee/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6a3ed70a-94ff-4809-9d6e-4532771366ee/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:03:12.212920092Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:03:12.214511896Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:03:12.214525326Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:03:12.274607655Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:03:12.484627976Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.811422574Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34156"}
{"msg":"region c813771a-b169-4a38-a026-6a40cfb0cb87 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.838560791Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.849198077Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions","method":"POST","req_id":"e89ec753-0423-47d3-a96a-3e52ebc8b337","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20395,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.849533381Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_6a3ed70a-94ff-4809-9d6e-4532771366ee/crucible/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.867408733Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6a3ed70a-94ff-4809-9d6e-4532771366ee/crucible/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.867444353Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c813771a-b169-4a38-a026-6a40cfb0cb87\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c813771a-b169-4a38-a026-6a40cfb0cb87\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.939766951Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"region":"c813771a-b169-4a38-a026-6a40cfb0cb87","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c813771a-b169-4a38-a026-6a40cfb0cb87\" \"--data\" \"/data/regions/c813771a-b169-4a38-a026-6a40cfb0cb87\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:04.942020894Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"region":"c813771a-b169-4a38-a026-6a40cfb0cb87","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.184745124Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions","method":"POST","req_id":"d1db1eb8-45f7-4cb8-9955-11722b31c3e6","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.328552057Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"region":"c813771a-b169-4a38-a026-6a40cfb0cb87","component":"worker"}
{"msg":"region c813771a-b169-4a38-a026-6a40cfb0cb87 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.328593568Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.328726259Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.329237187Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.339274194Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.362738047Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.370653533Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.383655403Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.386001277Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.386053668Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.386092698Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.393911523Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.401504814Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:05.876481061Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions","method":"POST","req_id":"f4bec0c1-b60c-4301-8b43-400e28e990c2","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.083930517Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","method":"GET","req_id":"0d4ed8f9-e611-48d2-8586-0a9964d65027","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/c813771a-b169-4a38-a026-6a40cfb0cb87\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.086737268Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"dataset is oxp_6a3ed70a-94ff-4809-9d6e-4532771366ee/crucible/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.096531682Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"region c813771a-b169-4a38-a026-6a40cfb0cb87 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.105001085Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.105200159Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.10527326Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","method":"DELETE","req_id":"7338820c-a82c-481f-a646-faf82d9aa0e3","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18581,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c813771a-b169-4a38-a026-6a40cfb0cb87 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.106406796Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.108140711Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","method":"GET","req_id":"77cd615b-d187-42d7-8bc0-af0a1f6ae408","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.111030254Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6a3ed70a-94ff-4809-9d6e-4532771366ee/crucible/regions/c813771a-b169-4a38-a026-6a40cfb0cb87\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.119916024Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"region":"c813771a-b169-4a38-a026-6a40cfb0cb87","component":"worker"}
{"msg":"region c813771a-b169-4a38-a026-6a40cfb0cb87 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.16960941Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-19T19:10:37.174004055Z","hostname":"oxz_crucible_bec029dc-952e-478a-b20d-7f0d85507875","pid":19668,"uri":"/crucible/0/regions/c813771a-b169-4a38-a026-6a40cfb0cb87","method":"GET","req_id":"76d880d7-2f49-498e-8791-d56f7d757377","remote_addr":"[fd00:1122:3344:101::a]:34156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":540,"response_code":"200"}