[ Jul 30 01:30:26 Enabled. ]
[ Jul 30 01:30:26 Rereading configuration. ]
[ Jul 30 01:30:27 Rereading configuration. ]
[ Jul 30 01:30:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_930e6a24-49fc-4e7b-9647-bae326f820db/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_930e6a24-49fc-4e7b-9647-bae326f820db/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.30482242Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.305997752Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.306010972Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.341863996Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.478676644Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.590797859Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34298"}
{"msg":"region 67015901-2b3e-40e6-87e4-522b77092b10 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.613957404Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.619055895Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.623195787Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions","method":"POST","req_id":"d9d4766d-9f04-426a-a2fc-44fa917e2194","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":15610,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_930e6a24-49fc-4e7b-9647-bae326f820db/crucible/regions/67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.630691205Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_930e6a24-49fc-4e7b-9647-bae326f820db/crucible/regions/67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.630711989Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"67015901-2b3e-40e6-87e4-522b77092b10\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/67015901-2b3e-40e6-87e4-522b77092b10\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.765080037Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"region":"67015901-2b3e-40e6-87e4-522b77092b10","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"67015901-2b3e-40e6-87e4-522b77092b10\" \"--data\" \"/data/regions/67015901-2b3e-40e6-87e4-522b77092b10\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.767299359Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"region":"67015901-2b3e-40e6-87e4-522b77092b10","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.9291202Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions","method":"POST","req_id":"997308c6-6b3e-4ad4-89cb-f9483ce93483","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":875,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.971155186Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"region":"67015901-2b3e-40e6-87e4-522b77092b10","component":"worker"}
{"msg":"region 67015901-2b3e-40e6-87e4-522b77092b10 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.971183477Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.971317158Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.971789003Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.976259168Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.993669075Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.997089499Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.003135861Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.005111291Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.005166131Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.005206812Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.008803198Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.012630997Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:44.400201778Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions","method":"POST","req_id":"1374f615-6719-42bd-aaca-11231001f108","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.01171143Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions/67015901-2b3e-40e6-87e4-522b77092b10","method":"GET","req_id":"06dd51e4-ec87-4073-8646-7ec9f41abfc2","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/67015901-2b3e-40e6-87e4-522b77092b10\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.013794332Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"dataset is oxp_930e6a24-49fc-4e7b-9647-bae326f820db/crucible/regions/67015901-2b3e-40e6-87e4-522b77092b10","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.023938354Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"region 67015901-2b3e-40e6-87e4-522b77092b10 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.032908536Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.033071257Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.033151688Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions/67015901-2b3e-40e6-87e4-522b77092b10","method":"DELETE","req_id":"2608ae19-25d3-4560-b044-1448678ad52a","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19414,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-67015901-2b3e-40e6-87e4-522b77092b10 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.034149118Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.035713784Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions/67015901-2b3e-40e6-87e4-522b77092b10","method":"GET","req_id":"eb96cc19-e5e0-4fde-8466-0be54bf144e8","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":473,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.038344201Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_930e6a24-49fc-4e7b-9647-bae326f820db/crucible/regions/67015901-2b3e-40e6-87e4-522b77092b10\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.047980498Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"region":"67015901-2b3e-40e6-87e4-522b77092b10","component":"worker"}
{"msg":"region 67015901-2b3e-40e6-87e4-522b77092b10 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.102478151Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.134403225Z","hostname":"oxz_crucible_66b1b7cd-2038-4e2d-b950-70a1c443cd33","pid":25370,"uri":"/crucible/0/regions/67015901-2b3e-40e6-87e4-522b77092b10","method":"GET","req_id":"ea39c6ad-8616-4557-a697-2d65d19478da","remote_addr":"[fd00:1122:3344:101::a]:34298","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":"200"}