[ Jan 22 09:21:47 Enabled. ] [ Jan 22 09:21:47 Rereading configuration. ] [ Jan 22 09:21:49 Rereading configuration. ] [ Jan 22 09:21:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5a89a118-2a6b-4435-9092-d6dd78aea2e3/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_5a89a118-2a6b-4435-9092-d6dd78aea2e3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.140286404Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.141234154Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.141244104Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.15949821Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.232448383Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.356555577Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43686"} {"msg":"region 32ee9385-fcbb-40a6-b049-cc923855563c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.385754671Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.390126568Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"uri":"/crucible/0/regions","method":"POST","req_id":"1e70be08-1eee-4b10-b623-faeb016968e6","remote_addr":"[fd00:1122:3344:101::c]:43686","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":12617,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.390681704Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5a89a118-2a6b-4435-9092-d6dd78aea2e3/crucible/regions/32ee9385-fcbb-40a6-b049-cc923855563c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.399842782Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5a89a118-2a6b-4435-9092-d6dd78aea2e3/crucible/regions/32ee9385-fcbb-40a6-b049-cc923855563c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.399884763Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"32ee9385-fcbb-40a6-b049-cc923855563c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/32ee9385-fcbb-40a6-b049-cc923855563c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.458074178Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"region":"32ee9385-fcbb-40a6-b049-cc923855563c","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"32ee9385-fcbb-40a6-b049-cc923855563c\" \"--data\" \"/data/regions/32ee9385-fcbb-40a6-b049-cc923855563c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.459979078Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"region":"32ee9385-fcbb-40a6-b049-cc923855563c","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.601607698Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"region":"32ee9385-fcbb-40a6-b049-cc923855563c","component":"worker"} {"msg":"region 32ee9385-fcbb-40a6-b049-cc923855563c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.601671479Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.60175631Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-32ee9385-fcbb-40a6-b049-cc923855563c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.602206855Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.607627753Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"uri":"/crucible/0/regions","method":"POST","req_id":"9733f408-ce51-457f-9b84-bd484cafd856","remote_addr":"[fd00:1122:3344:101::c]:43686","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-32ee9385-fcbb-40a6-b049-cc923855563c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.608476312Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.626535376Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-32ee9385-fcbb-40a6-b049-cc923855563c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.640697038Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/32ee9385-fcbb-40a6-b049-cc923855563c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.641216884Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.643174125Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.643234365Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.643278436Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.64835233Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.653483295Z","hostname":"oxz_crucible_34f00e5a-5f0c-4095-9441-c6a39fae032b","pid":22520,"component":"worker"}