[ Sep 10 21:24:04 Enabled. ]
[ Sep 10 21:24:04 Rereading configuration. ]
[ Sep 10 21:24:05 Rereading configuration. ]
[ Sep 10 21:24:07 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:07.292400654Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:07.299893655Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:07.299909025Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:07.32080044Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:07.450727777Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:44.95530556Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61980"}
{"msg":"region 2f9479f2-8c0d-4017-8f03-4929df589427 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:44.990397163Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:44.995011483Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions","method":"POST","req_id":"fc20df19-8b57-42ed-be3c-d4194685f53f","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24152,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:44.999218017Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible/regions/2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.012087941Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible/regions/2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.012164668Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2f9479f2-8c0d-4017-8f03-4929df589427\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2f9479f2-8c0d-4017-8f03-4929df589427\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.094241745Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"region":"2f9479f2-8c0d-4017-8f03-4929df589427","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2f9479f2-8c0d-4017-8f03-4929df589427\" \"--data\" \"/data/regions/2f9479f2-8c0d-4017-8f03-4929df589427\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.096349762Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"region":"2f9479f2-8c0d-4017-8f03-4929df589427","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.255289615Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"region":"2f9479f2-8c0d-4017-8f03-4929df589427","component":"worker"}
{"msg":"region 2f9479f2-8c0d-4017-8f03-4929df589427 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.255379916Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.255535319Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.256001187Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.266010611Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.292968502Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.300685167Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.316429681Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.318769482Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.318827563Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.318887904Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.331734606Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.344342758Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:45.351280149Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions","method":"POST","req_id":"9bd5c9dc-0710-4c93-acd9-40f4597c4b29","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":712,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.586707053Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions/2f9479f2-8c0d-4017-8f03-4929df589427","method":"GET","req_id":"20cd17a7-342e-42f7-874b-dddf0c0d1c55","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/2f9479f2-8c0d-4017-8f03-4929df589427\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.590429818Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"dataset is oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible/regions/2f9479f2-8c0d-4017-8f03-4929df589427","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.600950272Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"region 2f9479f2-8c0d-4017-8f03-4929df589427 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.611040998Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.611248692Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.611319923Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions/2f9479f2-8c0d-4017-8f03-4929df589427","method":"DELETE","req_id":"aba36ae3-335c-4e88-b92b-9193a105d385","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20919,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-2f9479f2-8c0d-4017-8f03-4929df589427 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.61231962Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.614831964Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions/2f9479f2-8c0d-4017-8f03-4929df589427","method":"GET","req_id":"7effb8ef-0026-4a95-8cd1-fc0f6b727488","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":719,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.620710387Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible/regions/2f9479f2-8c0d-4017-8f03-4929df589427\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.632654895Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"region":"2f9479f2-8c0d-4017-8f03-4929df589427","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.705776791Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions/2f9479f2-8c0d-4017-8f03-4929df589427","method":"GET","req_id":"8f772c03-f221-48e6-be48-c676d679602b","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":101,"response_code":"200"}
{"msg":"region 2f9479f2-8c0d-4017-8f03-4929df589427 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.710629766Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-10T21:24:46.780124518Z","hostname":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","pid":22329,"uri":"/crucible/0/regions/2f9479f2-8c0d-4017-8f03-4929df589427","method":"GET","req_id":"81bac9db-0495-4d9b-a137-290f9eea4d3d","remote_addr":"[fd00:1122:3344:101::c]:61980","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":61,"response_code":"200"}