[ Feb 22 18:31:16 Enabled. ]
[ Feb 22 18:31:16 Rereading configuration. ]
[ Feb 22 18:31:18 Rereading configuration. ]
[ Feb 22 18:31:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:29.680445234Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:29.684348541Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:29.684361541Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:29.997524488Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.717615997Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40037"}
{"msg":"region 57bc9c39-a26f-43ac-9b01-f66dfc51aebb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.796489026Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.799391412Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions","method":"POST","req_id":"3bd8c09e-7766-43d3-957d-143be916a425","remote_addr":"[fd00:1122:3344:101::a]:40037","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":14377,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.799996404Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.809556841Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.809591651Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"57bc9c39-a26f-43ac-9b01-f66dfc51aebb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.875736521Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"region":"57bc9c39-a26f-43ac-9b01-f66dfc51aebb","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.011149732Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"region":"57bc9c39-a26f-43ac-9b01-f66dfc51aebb","component":"worker"}
{"msg":"region 57bc9c39-a26f-43ac-9b01-f66dfc51aebb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.011218923Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.011305105Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.011806665Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.017481976Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.03567879Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.039043566Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.050772575Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.052801694Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.052859155Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.052899746Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.058093857Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.062231968Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.095816879Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions","method":"POST","req_id":"2aa17fac-7039-4cb9-a845-742daa494da9","remote_addr":"[fd00:1122:3344:101::a]:40037","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.607270092Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39982"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.609339822Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","method":"GET","req_id":"069ebc6f-8c76-4a8d-a490-e8d2cc381f65","remote_addr":"[fd00:1122:3344:101::a]:39982","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2043,"response_code":"200"}
{"msg":"path is \"/data/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.609463185Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.619078863Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"datafile"}
{"msg":"region 57bc9c39-a26f-43ac-9b01-f66dfc51aebb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.626714352Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.626898605Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.626961126Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","method":"DELETE","req_id":"bb77c15c-1b2b-4c57-bde7-ad6079a84652","remote_addr":"[fd00:1122:3344:101::a]:39982","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":17512,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.62713704Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","method":"GET","req_id":"cd7c0ece-ee5a-454a-926d-d69747f7021b","remote_addr":"[fd00:1122:3344:101::a]:39982","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-57bc9c39-a26f-43ac-9b01-f66dfc51aebb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.628018617Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.631699499Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.641476389Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"region":"57bc9c39-a26f-43ac-9b01-f66dfc51aebb","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.683434987Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","method":"GET","req_id":"d73707fe-6313-4036-af3b-bdfb2cd2ef0d","remote_addr":"[fd00:1122:3344:101::a]:39982","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"region 57bc9c39-a26f-43ac-9b01-f66dfc51aebb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.69073782Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.747472887Z","hostname":"oxz_crucible_d64def60-1829-4c6d-9f91-15ca4b937ea0","pid":26527,"uri":"/crucible/0/regions/57bc9c39-a26f-43ac-9b01-f66dfc51aebb","method":"GET","req_id":"e3e43939-8079-4fab-a616-c430a5eb12e3","remote_addr":"[fd00:1122:3344:101::a]:39982","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":782,"response_code":"200"}