[ Jun 24 21:11:28 Enabled. ]
[ Jun 24 21:11:28 Rereading configuration. ]
[ Jun 24 21:11:29 Rereading configuration. ]
[ Jun 24 21:11:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6cc0b202-3570-4e11-98e5-e2e8454304ed/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6cc0b202-3570-4e11-98e5-e2e8454304ed/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:11:31.754597802Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:11:31.757756884Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:11:31.757780554Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:11:31.781563769Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:11:31.854649958Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.075565793Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:58460"}
{"msg":"region 267bdcac-c07a-4b31-b988-3bfd8a06ebd0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.094701797Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.099793739Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.103819343Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"uri":"/crucible/0/regions","method":"POST","req_id":"878159c9-e588-4445-ab9d-91032f74dba1","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":13025,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_6cc0b202-3570-4e11-98e5-e2e8454304ed/crucible/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.10949724Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6cc0b202-3570-4e11-98e5-e2e8454304ed/crucible/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.109551671Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"267bdcac-c07a-4b31-b988-3bfd8a06ebd0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.194898836Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"region":"267bdcac-c07a-4b31-b988-3bfd8a06ebd0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"267bdcac-c07a-4b31-b988-3bfd8a06ebd0\" \"--data\" \"/data/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.196858177Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"region":"267bdcac-c07a-4b31-b988-3bfd8a06ebd0","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.319079485Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"region":"267bdcac-c07a-4b31-b988-3bfd8a06ebd0","component":"worker"}
{"msg":"region 267bdcac-c07a-4b31-b988-3bfd8a06ebd0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.319165297Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.319390483Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.319789233Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.324237948Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.33512888Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.340818657Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.347364826Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.349432309Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.349509351Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.349572173Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.354553161Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.359336605Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:26.390274164Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"uri":"/crucible/0/regions","method":"POST","req_id":"56c629e0-7abb-4670-a212-012fa9e8c24e","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":694,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.382363289Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"uri":"/crucible/0/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","method":"GET","req_id":"e4647034-4144-4273-9f0b-5d97623e0dfd","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.385203672Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"dataset is oxp_6cc0b202-3570-4e11-98e5-e2e8454304ed/crucible/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.395319324Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"region 267bdcac-c07a-4b31-b988-3bfd8a06ebd0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.404013809Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.404308966Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-267bdcac-c07a-4b31-b988-3bfd8a06ebd0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.405580029Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.406297368Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"uri":"/crucible/0/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","method":"DELETE","req_id":"c56dd0e7-e70d-41e0-8aef-5eca72b92acd","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21134,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.409970663Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"uri":"/crucible/0/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","method":"GET","req_id":"9b874830-ddd3-41b0-b268-30fa92902dc7","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.410531497Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6cc0b202-3570-4e11-98e5-e2e8454304ed/crucible/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.420285579Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"region":"267bdcac-c07a-4b31-b988-3bfd8a06ebd0","component":"worker"}
{"msg":"region 267bdcac-c07a-4b31-b988-3bfd8a06ebd0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.484724524Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-24T21:12:27.527583071Z","hostname":"oxz_crucible_a22cbe0e-1b82-47ce-b737-216bcb321f12","pid":21404,"uri":"/crucible/0/regions/267bdcac-c07a-4b31-b988-3bfd8a06ebd0","method":"GET","req_id":"c216b7eb-1a85-4ba4-a6d7-cac7e41bc3b2","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":627,"response_code":"200"}