[ Oct 11 16:01:12 Enabled. ]
[ Oct 11 16:01:12 Rereading configuration. ]
[ Oct 11 16:01:15 Rereading configuration. ]
[ Oct 11 16:01:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:01:18.312383511Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:01:18.313704684Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:01:18.313712554Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:01:18.325916699Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:01:18.438334693Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.565493565Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45134"}
{"msg":"region db2b60ac-3a85-4a69-b171-e63c490b49c0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.596894427Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.601204663Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions","method":"POST","req_id":"aee4521d-c71c-453f-88df-6d20cbeb77d3","remote_addr":"[fd00:1122:3344:101::a]:45134","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":12117,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.602287592Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crucible/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.617683222Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crucible/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.617717943Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"db2b60ac-3a85-4a69-b171-e63c490b49c0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.723056833Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"region":"db2b60ac-3a85-4a69-b171-e63c490b49c0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"db2b60ac-3a85-4a69-b171-e63c490b49c0\" \"--data\" \"/data/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.72686721Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"region":"db2b60ac-3a85-4a69-b171-e63c490b49c0","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:25.910591307Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions","method":"POST","req_id":"a25bb5e1-3eeb-484e-8232-71ab90d9d447","remote_addr":"[fd00:1122:3344:101::a]:45134","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.06046563Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"region":"db2b60ac-3a85-4a69-b171-e63c490b49c0","component":"worker"}
{"msg":"region db2b60ac-3a85-4a69-b171-e63c490b49c0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.060514521Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.060630173Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.063011175Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.078314294Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.114266875Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.127506628Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.141612126Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.141678467Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.141735588Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.141793959Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.167736835Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.188187414Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:02:26.397679864Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions","method":"POST","req_id":"2302f13e-c0c0-4d44-b3a3-421617091f1a","remote_addr":"[fd00:1122:3344:101::a]:45134","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":723,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.043403855Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35766"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.047369104Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","method":"GET","req_id":"f130ccb0-4339-43a1-8e3a-94a950bacfe3","remote_addr":"[fd00:1122:3344:101::a]:35766","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":3744,"response_code":"200"}
{"msg":"path is \"/data/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.051030499Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"dataset is oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crucible/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.063914135Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"region db2b60ac-3a85-4a69-b171-e63c490b49c0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.074064353Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.074741195Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-db2b60ac-3a85-4a69-b171-e63c490b49c0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.075888845Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.077165058Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","method":"DELETE","req_id":"5624f218-f1a9-4319-87e3-0cb9f8d115b7","remote_addr":"[fd00:1122:3344:101::a]:35766","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26173,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.081022706Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","method":"GET","req_id":"50b5940b-1c9e-496b-afe4-3cb88c96c3b2","remote_addr":"[fd00:1122:3344:101::a]:35766","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.089270251Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crucible/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.100820783Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"region":"db2b60ac-3a85-4a69-b171-e63c490b49c0","component":"worker"}
{"msg":"region db2b60ac-3a85-4a69-b171-e63c490b49c0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.186448478Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-11T16:05:34.222802836Z","hostname":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","pid":24507,"uri":"/crucible/0/regions/db2b60ac-3a85-4a69-b171-e63c490b49c0","method":"GET","req_id":"94261e84-e8a2-4a48-bcc3-e30d381be1ef","remote_addr":"[fd00:1122:3344:101::a]:35766","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":65,"response_code":"200"}