[ Nov  9 02:46:17 Enabled. ]
[ Nov  9 02:46:17 Rereading configuration. ]
[ Nov  9 02:46:19 Rereading configuration. ]
[ Nov  9 02:46:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.035580923Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.039557846Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.039577086Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.0634648Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.207583998Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.12763834Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39404"}
{"msg":"region 88627561-d2db-4f09-a8d1-a3758fdbd3d2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.13490047Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.135239657Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"uri":"/crucible/0/regions","method":"POST","req_id":"d46a0e44-dcda-431f-a7e7-ac08a0dd41ba","remote_addr":"[fd00:1122:3344:101::a]:39404","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2873,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.136520633Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.147218484Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.147250615Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"88627561-d2db-4f09-a8d1-a3758fdbd3d2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.213260279Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"region":"88627561-d2db-4f09-a8d1-a3758fdbd3d2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"88627561-d2db-4f09-a8d1-a3758fdbd3d2\" \"--data\" \"/data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.213380082Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"region":"88627561-d2db-4f09-a8d1-a3758fdbd3d2","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.435702806Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"region":"88627561-d2db-4f09-a8d1-a3758fdbd3d2","component":"worker"}
{"msg":"region 88627561-d2db-4f09-a8d1-a3758fdbd3d2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.435768427Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.435854689Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.436321439Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.440570586Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.45186465Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.455097967Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.455861942Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.455996155Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.456043146Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.456082237Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.459848012Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.463858888Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.478150273Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"uri":"/crucible/0/regions","method":"POST","req_id":"99d41c88-f248-4568-ba6b-055c5c601e14","remote_addr":"[fd00:1122:3344:101::a]:39404","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.696088975Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64932"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.703677622Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"uri":"/crucible/0/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","method":"GET","req_id":"8a88c7b0-fd51-485d-899c-b8d9f828b351","remote_addr":"[fd00:1122:3344:101::a]:64932","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":7493,"response_code":"200"}
{"msg":"path is \"/data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.70644861Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"dataset is oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.715942616Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"snapshot is oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2@37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.724968002Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\", \"37f3fb8c-b87c-443f-a1cf-79a4c951c424\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.724999523Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.736327397Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"uri":"/crucible/0/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2/snapshots/37f3fb8c-b87c-443f-a1cf-79a4c951c424","method":"GET","req_id":"df01897e-1f1b-4fca-a7ca-153be1406d94","remote_addr":"[fd00:1122:3344:101::a]:64932","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29888,"response_code":"200"}
{"msg":"path is \"/data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.739459392Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"dataset is oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.748731443Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"snapshot is oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2@37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.756750989Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\", \"37f3fb8c-b87c-443f-a1cf-79a4c951c424\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.75678122Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 88627561-d2db-4f09-a8d1-a3758fdbd3d2-37f3fb8c-b87c-443f-a1cf-79a4c951c424 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.764495289Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"applying SMF actions for region 88627561-d2db-4f09-a8d1-a3758fdbd3d2 running snapshot 37f3fb8c-b87c-443f-a1cf-79a4c951c424 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.765048571Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.765075671Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"uri":"/crucible/0/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2/snapshots/37f3fb8c-b87c-443f-a1cf-79a4c951c424/run","method":"POST","req_id":"edcf13ca-e8e5-475a-acca-b70ce7a850e3","remote_addr":"[fd00:1122:3344:101::a]:64932","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25647,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-88627561-d2db-4f09-a8d1-a3758fdbd3d2-37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.772142447Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-88627561-d2db-4f09-a8d1-a3758fdbd3d2-37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.778799875Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-88627561-d2db-4f09-a8d1-a3758fdbd3d2-37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.801231539Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2/.zfs/snapshot/37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.801525845Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.801573066Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.801615216Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.801651247Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.801695468Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.808387026Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.815267289Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"worker"}
{"msg":"region 88627561-d2db-4f09-a8d1-a3758fdbd3d2 running snapshot 37f3fb8c-b87c-443f-a1cf-79a4c951c424 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.815282219Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"path is \"/data/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.000546448Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"dataset is oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.009548304Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"datafile"}
{"msg":"snapshot is oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2@37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.017529759Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crucible/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2\", \"37f3fb8c-b87c-443f-a1cf-79a4c951c424\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.01756981Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.024891143Z","hostname":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","pid":28542,"uri":"/crucible/0/regions/88627561-d2db-4f09-a8d1-a3758fdbd3d2/snapshots/37f3fb8c-b87c-443f-a1cf-79a4c951c424/run","method":"POST","req_id":"a1b14153-d57b-4aa4-9f38-d4c89fd97a89","remote_addr":"[fd00:1122:3344:101::a]:64932","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24427,"response_code":"200"}