[ Aug 27 19:02:02 Enabled. ]
[ Aug 27 19:02:02 Rereading configuration. ]
[ Aug 27 19:02:07 Rereading configuration. ]
[ Aug 27 19:02:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_030f20e2-2758-4a47-92fd-4adaf0953899/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_030f20e2-2758-4a47-92fd-4adaf0953899/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:17.749965004Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:17.750778231Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:17.750791501Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:17.764981511Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:17.952051959Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:48.753185481Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43387"}
{"msg":"region cf650e85-58b0-4977-82a5-843b7063cd58 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:48.821005476Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:48.834092043Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions","method":"POST","req_id":"02c2cf75-13f7-40ba-8f8d-4a767a3ff990","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":42964,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:48.834919989Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_030f20e2-2758-4a47-92fd-4adaf0953899/crucible/regions/cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:48.860247646Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_030f20e2-2758-4a47-92fd-4adaf0953899/crucible/regions/cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:48.860279487Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cf650e85-58b0-4977-82a5-843b7063cd58\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cf650e85-58b0-4977-82a5-843b7063cd58\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.002159853Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"region":"cf650e85-58b0-4977-82a5-843b7063cd58","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cf650e85-58b0-4977-82a5-843b7063cd58\" \"--data\" \"/data/regions/cf650e85-58b0-4977-82a5-843b7063cd58\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.004331978Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"region":"cf650e85-58b0-4977-82a5-843b7063cd58","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.050377857Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions","method":"POST","req_id":"1d1326b0-bf05-47f4-8efc-2bc7be5574c4","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.361258943Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions","method":"POST","req_id":"f7ef7512-bcca-4ca6-a6f9-9231789c2ef1","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":43,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.940818221Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"region":"cf650e85-58b0-4977-82a5-843b7063cd58","component":"worker"}
{"msg":"region cf650e85-58b0-4977-82a5-843b7063cd58 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.940900473Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.940970245Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.941367453Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:49.958600744Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.015420614Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.042701241Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.067968177Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.079552383Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.079607394Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.079640615Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.087087607Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.100272586Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:50.53551142Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions","method":"POST","req_id":"f870d638-768b-4d85-8294-991c625dfb0d","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.407643134Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions/cf650e85-58b0-4977-82a5-843b7063cd58","method":"GET","req_id":"45326113-6219-4963-a5d2-1462ae64cb45","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":647,"response_code":"200"}
{"msg":"path is \"/data/regions/cf650e85-58b0-4977-82a5-843b7063cd58\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.410229596Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"dataset is oxp_030f20e2-2758-4a47-92fd-4adaf0953899/crucible/regions/cf650e85-58b0-4977-82a5-843b7063cd58","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.421274861Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"region cf650e85-58b0-4977-82a5-843b7063cd58 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.432039331Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.4321969Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.432263548Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions/cf650e85-58b0-4977-82a5-843b7063cd58","method":"DELETE","req_id":"845b0ae9-aa5a-4688-80f7-928554ecd7d4","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":22016,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-cf650e85-58b0-4977-82a5-843b7063cd58 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.433278755Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.435183451Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions/cf650e85-58b0-4977-82a5-843b7063cd58","method":"GET","req_id":"634a88e8-ca13-488c-95ef-5afbc16c316f","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.452501353Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_030f20e2-2758-4a47-92fd-4adaf0953899/crucible/regions/cf650e85-58b0-4977-82a5-843b7063cd58\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.463126765Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"region":"cf650e85-58b0-4977-82a5-843b7063cd58","component":"worker"}
{"msg":"region cf650e85-58b0-4977-82a5-843b7063cd58 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.51460887Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:09:18.544367764Z","hostname":"oxz_crucible_064b1447-702f-4f88-b0e7-1552e0564dfb","pid":25024,"uri":"/crucible/0/regions/cf650e85-58b0-4977-82a5-843b7063cd58","method":"GET","req_id":"ae0d67a3-fd1f-4dd0-acf0-dec45c7c165a","remote_addr":"[fd00:1122:3344:101::a]:43387","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":57,"response_code":"200"}