[ Aug 22 17:49:11 Enabled. ]
[ Aug 22 17:49:11 Rereading configuration. ]
[ Aug 22 17:49:16 Rereading configuration. ]
[ Aug 22 17:49:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_18a9b981-177e-4a7a-afbe-74c54d247869/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_18a9b981-177e-4a7a-afbe-74c54d247869/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.485829976Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.48675358Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.48676331Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.504486142Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.647221179Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:30.901503478Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51062"}
{"msg":"region f5ed6677-21e4-4913-83fe-d2eb913a3b66 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:30.928219406Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:30.935006731Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions","method":"POST","req_id":"95755256-1d29-4a16-9052-9c814e24939b","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":16007,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:30.935535114Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_18a9b981-177e-4a7a-afbe-74c54d247869/crucible/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:30.95029019Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_18a9b981-177e-4a7a-afbe-74c54d247869/crucible/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:30.950325981Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f5ed6677-21e4-4913-83fe-d2eb913a3b66\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.010336731Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"region":"f5ed6677-21e4-4913-83fe-d2eb913a3b66","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f5ed6677-21e4-4913-83fe-d2eb913a3b66\" \"--data\" \"/data/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.012692993Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"region":"f5ed6677-21e4-4913-83fe-d2eb913a3b66","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.184234939Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions","method":"POST","req_id":"4d851cfb-88b2-447c-b436-9dbc989b46b0","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":753,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.308581442Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"region":"f5ed6677-21e4-4913-83fe-d2eb913a3b66","component":"worker"}
{"msg":"region f5ed6677-21e4-4913-83fe-d2eb913a3b66 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.308630192Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.308715733Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.309139115Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.313889209Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.33330332Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.344830169Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.347874525Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.350462558Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.350512938Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.350551559Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.354444579Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.358943872Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:31.847204905Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions","method":"POST","req_id":"3c1278de-e8a1-4d69-91ad-9d407bb59b16","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.525759015Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","method":"GET","req_id":"7f44b6f0-1b35-4a55-aa23-ac5cab4630c2","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.529669038Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"dataset is oxp_18a9b981-177e-4a7a-afbe-74c54d247869/crucible/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.541022954Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"region f5ed6677-21e4-4913-83fe-d2eb913a3b66 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.551691899Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.55187569Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.55195697Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","method":"DELETE","req_id":"9d39ed44-e4a1-42f4-8e7d-6dc78654c3b3","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":22334,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f5ed6677-21e4-4913-83fe-d2eb913a3b66 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.552957636Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.556411313Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","method":"GET","req_id":"a1a15e2a-95e2-4b30-bb02-736ddead6b5a","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.558838296Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_18a9b981-177e-4a7a-afbe-74c54d247869/crucible/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.569138849Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"region":"f5ed6677-21e4-4913-83fe-d2eb913a3b66","component":"worker"}
{"msg":"region f5ed6677-21e4-4913-83fe-d2eb913a3b66 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.624180164Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:55:56.708817831Z","hostname":"oxz_crucible_b50125ad-86a9-4ea3-a948-c19cf7285d0a","pid":22243,"uri":"/crucible/0/regions/f5ed6677-21e4-4913-83fe-d2eb913a3b66","method":"GET","req_id":"d3b89ef3-7667-4d59-a651-007fb32a637b","remote_addr":"[fd00:1122:3344:101::a]:51062","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":87,"response_code":"200"}