[ Jun 20 21:03:50 Enabled. ]
[ Jun 20 21:03:50 Rereading configuration. ]
[ Jun 20 21:03:51 Rereading configuration. ]
[ Jun 20 21:03:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.058923726Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.059329673Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.059340023Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.077766588Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:03:54.167391022Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.169304323Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43161"}
{"msg":"region 47d0a5b7-051f-467c-b049-f73c0df12f57 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.204874031Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.2117772Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions","method":"POST","req_id":"c094d02c-ec10-458c-a90f-efdc4dec28dd","remote_addr":"[fd00:1122:3344:101::a]:43161","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":15385,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.21180307Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.223427449Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.2234641Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"47d0a5b7-051f-467c-b049-f73c0df12f57\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.271117845Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"47d0a5b7-051f-467c-b049-f73c0df12f57","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"47d0a5b7-051f-467c-b049-f73c0df12f57\" \"--data\" \"/data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.273568117Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"47d0a5b7-051f-467c-b049-f73c0df12f57","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.551082555Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"47d0a5b7-051f-467c-b049-f73c0df12f57","component":"worker"}
{"msg":"region 47d0a5b7-051f-467c-b049-f73c0df12f57 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.551138146Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.551215167Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.551643255Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions","method":"POST","req_id":"5e841ed4-7b7c-4825-b635-2bfde5b1deda","remote_addr":"[fd00:1122:3344:101::a]:43161","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"creating missing downstairs instance downstairs-47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.551663965Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.55722962Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.568644175Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.573364666Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.58173984Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.584209242Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.584269483Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.584310554Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.588127409Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:05:06.594825944Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.031588532Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57540"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.036757081Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","method":"GET","req_id":"db8a2f40-2b58-42e9-83cb-ca973da62c87","remote_addr":"[fd00:1122:3344:101::a]:57540","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":5142,"response_code":"200"}
{"msg":"path is \"/data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.036974025Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"dataset is oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.046361765Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"snapshot is oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57@1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.056371497Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57\", \"1754a985-200c-4441-a679-cc9cc7979745\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.056395997Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.070279965Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/47d0a5b7-051f-467c-b049-f73c0df12f57/snapshots/1754a985-200c-4441-a679-cc9cc7979745","method":"GET","req_id":"03b91a85-89bf-452e-9408-ee5e1e3649e4","remote_addr":"[fd00:1122:3344:101::a]:57540","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":33334,"response_code":"200"}
{"msg":"path is \"/data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.070451308Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"dataset is oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.078731739Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"snapshot is oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57@1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.08694187Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/47d0a5b7-051f-467c-b049-f73c0df12f57\", \"1754a985-200c-4441-a679-cc9cc7979745\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.0869656Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 47d0a5b7-051f-467c-b049-f73c0df12f57-1754a985-200c-4441-a679-cc9cc7979745 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.095422845Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"applying SMF actions for region 47d0a5b7-051f-467c-b049-f73c0df12f57 running snapshot 1754a985-200c-4441-a679-cc9cc7979745 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.096341151Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.096380851Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/47d0a5b7-051f-467c-b049-f73c0df12f57/snapshots/1754a985-200c-4441-a679-cc9cc7979745/run","method":"POST","req_id":"0abe9059-7782-45b0-a86e-db94ab50acc2","remote_addr":"[fd00:1122:3344:101::a]:57540","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25956,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.100642324Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.100736166Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.100820947Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.100828097Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.100871168Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.100919419Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.101009231Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.101071822Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.101119862Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.101176294Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.111123344Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-47d0a5b7-051f-467c-b049-f73c0df12f57-1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.111606152Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-47d0a5b7-051f-467c-b049-f73c0df12f57-1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.119468816Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-47d0a5b7-051f-467c-b049-f73c0df12f57-1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.141157598Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57/.zfs/snapshot/1754a985-200c-4441-a679-cc9cc7979745","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.141438392Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.141492973Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.141535354Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.141575565Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.141627776Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.147380384Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.154427495Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"region 47d0a5b7-051f-467c-b049-f73c0df12f57 running snapshot 1754a985-200c-4441-a679-cc9cc7979745 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:19.154450615Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.336766812Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51131"}
{"msg":"region 90164b69-e614-45c0-a835-b11bbae4d96d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.336931045Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.337095168Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions","method":"POST","req_id":"9fe86927-a514-40dd-8779-f7390cc8b3c8","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":203,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.337121168Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.346776703Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.346809334Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"90164b69-e614-45c0-a835-b11bbae4d96d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/90164b69-e614-45c0-a835-b11bbae4d96d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.416770161Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"90164b69-e614-45c0-a835-b11bbae4d96d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"90164b69-e614-45c0-a835-b11bbae4d96d\" \"--data\" \"/data/regions/90164b69-e614-45c0-a835-b11bbae4d96d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.416836152Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"90164b69-e614-45c0-a835-b11bbae4d96d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.529946367Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions","method":"POST","req_id":"b3ed5a80-b51d-4c22-a1aa-6d72e8cdba01","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.539608622Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"90164b69-e614-45c0-a835-b11bbae4d96d","component":"worker"}
{"msg":"region 90164b69-e614-45c0-a835-b11bbae4d96d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.539651053Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.539798146Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.540454207Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.540544209Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.54062557Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.54063245Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.540676401Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.540727652Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.541118758Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.541179369Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.54123058Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.541280801Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.548039527Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.548476424Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.558348183Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.578192013Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.584145235Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.58448625Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.584544671Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.584587332Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.584625863Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.591648013Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:22.60080616Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:23.218709244Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions","method":"POST","req_id":"61c2f6c3-c1bb-4228-b552-33f66fdd5da0","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":589,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.684745382Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/90164b69-e614-45c0-a835-b11bbae4d96d","method":"GET","req_id":"e8b5b6b6-edd5-4718-87a4-9c53565b2070","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"path is \"/data/regions/90164b69-e614-45c0-a835-b11bbae4d96d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.687476458Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"dataset is oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/90164b69-e614-45c0-a835-b11bbae4d96d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.697974398Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"region 90164b69-e614-45c0-a835-b11bbae4d96d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.705916014Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.706145658Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.708310325Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/90164b69-e614-45c0-a835-b11bbae4d96d","method":"DELETE","req_id":"2715ec44-490f-4898-bba2-53a430f7d0c4","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20846,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-90164b69-e614-45c0-a835-b11bbae4d96d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.709442884Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.710789227Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/90164b69-e614-45c0-a835-b11bbae4d96d","method":"GET","req_id":"4a9c6287-0092-4290-949e-58fa36569a33","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.713932881Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714032733Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714118104Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714129444Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714167905Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714213986Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47d0a5b7-051f-467c-b049-f73c0df12f57","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714526191Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714607893Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714659213Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.714705804Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.721616022Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.722978586Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible/regions/90164b69-e614-45c0-a835-b11bbae4d96d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.732798744Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"region":"90164b69-e614-45c0-a835-b11bbae4d96d","component":"worker"}
{"msg":"region 90164b69-e614-45c0-a835-b11bbae4d96d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.785609077Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-20T21:10:47.816987454Z","hostname":"oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4","pid":22529,"uri":"/crucible/0/regions/90164b69-e614-45c0-a835-b11bbae4d96d","method":"GET","req_id":"f18a222e-a522-4c68-88ab-e25122751568","remote_addr":"[fd00:1122:3344:101::a]:51131","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":"200"}