[ Aug 1 01:16:18 Enabled. ] [ Aug 1 01:16:18 Rereading configuration. ] [ Aug 1 01:16:20 Rereading configuration. ] [ Aug 1 01:16:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.654627581Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544} {"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.675780678Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.675955561Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.695650832Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.916139672Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.207025842Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63398"} {"msg":"region 8436e497-e8fe-400f-bc5b-d1131767ee32 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.231985197Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.242888772Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.242910502Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"uri":"/crucible/0/regions","method":"POST","req_id":"e2c28afc-e83b-4087-9bc2-6b6afb96a6f9","remote_addr":"[fd00:1122:3344:101::a]:63398","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20781,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.256652687Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.256684998Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"8436e497-e8fe-400f-bc5b-d1131767ee32\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.316494024Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"region":"8436e497-e8fe-400f-bc5b-d1131767ee32","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8436e497-e8fe-400f-bc5b-d1131767ee32\" \"--data\" \"/data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.318890856Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"region":"8436e497-e8fe-400f-bc5b-d1131767ee32","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.457348913Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"region":"8436e497-e8fe-400f-bc5b-d1131767ee32","component":"worker"} {"msg":"region 8436e497-e8fe-400f-bc5b-d1131767ee32 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.457420575Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.457502256Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.457899563Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.463640235Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.479637381Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.482442211Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.483846906Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.489104829Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.48915516Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.489191911Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.496540602Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.503397334Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.541668436Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"uri":"/crucible/0/regions","method":"POST","req_id":"baacfd5c-c752-4220-ae11-f1da84f45678","remote_addr":"[fd00:1122:3344:101::a]:63398","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.355316826Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41088"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.358178087Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"uri":"/crucible/0/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","method":"GET","req_id":"b79d9d63-89d3-4dbd-8ddb-2cd56a8009d7","remote_addr":"[fd00:1122:3344:101::a]:41088","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2797,"response_code":"200"} {"msg":"path is \"/data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.362389392Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"dataset is oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.375578807Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"snapshot is oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32@88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.386892768Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32\", \"88052840-5a63-42fd-bb55-9316b8dcf787\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.386930759Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.401212224Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"uri":"/crucible/0/regions/8436e497-e8fe-400f-bc5b-d1131767ee32/snapshots/88052840-5a63-42fd-bb55-9316b8dcf787","method":"GET","req_id":"a993262c-750b-4d7c-97ee-d0f331388494","remote_addr":"[fd00:1122:3344:101::a]:41088","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":42801,"response_code":"200"} {"msg":"path is \"/data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.401452878Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"dataset is oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.414883627Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"snapshot is oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32@88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.427516992Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_31306bab-fd9a-43f3-a777-af05f3e098be/crucible/regions/8436e497-e8fe-400f-bc5b-d1131767ee32\", \"88052840-5a63-42fd-bb55-9316b8dcf787\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.427570573Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 8436e497-e8fe-400f-bc5b-d1131767ee32-88052840-5a63-42fd-bb55-9316b8dcf787 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.440921491Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"} {"msg":"applying SMF actions for region 8436e497-e8fe-400f-bc5b-d1131767ee32 running snapshot 88052840-5a63-42fd-bb55-9316b8dcf787 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.441682575Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.441772197Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"uri":"/crucible/0/regions/8436e497-e8fe-400f-bc5b-d1131767ee32/snapshots/88052840-5a63-42fd-bb55-9316b8dcf787/run","method":"POST","req_id":"8b0ade51-739b-4a2a-b2e8-e7de6763b5d3","remote_addr":"[fd00:1122:3344:101::a]:41088","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":40359,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.452550349Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.452686681Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.452810853Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.452820874Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.452886225Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.452954816Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.453098928Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.45319124Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.453264982Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.453339853Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.462219611Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-8436e497-e8fe-400f-bc5b-d1131767ee32-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.462889823Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-8436e497-e8fe-400f-bc5b-d1131767ee32-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.471071349Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-8436e497-e8fe-400f-bc5b-d1131767ee32-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.487241827Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8436e497-e8fe-400f-bc5b-d1131767ee32/.zfs/snapshot/88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.487730976Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.487817367Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.487878668Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.48794632Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.488028471Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.4930352Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.49805713Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"worker"} {"msg":"region 8436e497-e8fe-400f-bc5b-d1131767ee32 running snapshot 88052840-5a63-42fd-bb55-9316b8dcf787 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.49808163Z","hostname":"oxz_crucible_81aa442d-0253-4e50-b6d9-8baf861c6e6e","pid":23544,"component":"datafile"}