BPFTable.cc 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. /*
  2. * Copyright (c) 2016 Facebook, Inc.
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include <fcntl.h>
  17. #include <linux/elf.h>
  18. #include <linux/perf_event.h>
  19. #include <sys/epoll.h>
  20. #include <unistd.h>
  21. #include <cerrno>
  22. #include <cinttypes>
  23. #include <cstdint>
  24. #include <cstring>
  25. #include <iostream>
  26. #include <memory>
  27. #include "BPFTable.h"
  28. #include "bcc_exception.h"
  29. #include "bcc_syms.h"
  30. #include "common.h"
  31. #include "file_desc.h"
  32. #include "libbpf.h"
  33. #include "perf_reader.h"
  34. namespace ebpf {
  35. BPFTable::BPFTable(const TableDesc& desc) : BPFTableBase<void, void>(desc) {}
  36. StatusTuple BPFTable::get_value(const std::string& key_str,
  37. std::string& value_str) {
  38. char key[desc.key_size];
  39. char value[desc.leaf_size];
  40. StatusTuple r(0);
  41. r = string_to_key(key_str, key);
  42. if (!r.ok())
  43. return r;
  44. if (!lookup(key, value))
  45. return StatusTuple(-1, "error getting value");
  46. return leaf_to_string(value, value_str);
  47. }
  48. StatusTuple BPFTable::get_value(const std::string& key_str,
  49. std::vector<std::string>& value_str) {
  50. size_t ncpus = get_possible_cpus().size();
  51. char key[desc.key_size];
  52. char value[desc.leaf_size * ncpus];
  53. StatusTuple r(0);
  54. r = string_to_key(key_str, key);
  55. if (!r.ok())
  56. return r;
  57. if (!lookup(key, value))
  58. return StatusTuple(-1, "error getting value");
  59. value_str.resize(ncpus);
  60. for (size_t i = 0; i < ncpus; i++) {
  61. r = leaf_to_string(value + i * desc.leaf_size, value_str.at(i));
  62. if (!r.ok())
  63. return r;
  64. }
  65. return StatusTuple::OK();
  66. }
  67. StatusTuple BPFTable::update_value(const std::string& key_str,
  68. const std::string& value_str) {
  69. char key[desc.key_size];
  70. char value[desc.leaf_size];
  71. StatusTuple r(0);
  72. r = string_to_key(key_str, key);
  73. if (!r.ok())
  74. return r;
  75. r = string_to_leaf(value_str, value);
  76. if (!r.ok())
  77. return r;
  78. if (!update(key, value))
  79. return StatusTuple(-1, "error updating element");
  80. return StatusTuple::OK();
  81. }
  82. StatusTuple BPFTable::update_value(const std::string& key_str,
  83. const std::vector<std::string>& value_str) {
  84. size_t ncpus = get_possible_cpus().size();
  85. char key[desc.key_size];
  86. char value[desc.leaf_size * ncpus];
  87. StatusTuple r(0);
  88. r = string_to_key(key_str, key);
  89. if (!r.ok())
  90. return r;
  91. if (value_str.size() != ncpus)
  92. return StatusTuple(-1, "bad value size");
  93. for (size_t i = 0; i < ncpus; i++) {
  94. r = string_to_leaf(value_str.at(i), value + i * desc.leaf_size);
  95. if (!r.ok())
  96. return r;
  97. }
  98. if (!update(key, value))
  99. return StatusTuple(-1, "error updating element");
  100. return StatusTuple::OK();
  101. }
  102. StatusTuple BPFTable::remove_value(const std::string& key_str) {
  103. char key[desc.key_size];
  104. StatusTuple r(0);
  105. r = string_to_key(key_str, key);
  106. if (!r.ok())
  107. return r;
  108. if (!remove(key))
  109. return StatusTuple(-1, "error removing element");
  110. return StatusTuple::OK();
  111. }
  112. StatusTuple BPFTable::clear_table_non_atomic() {
  113. if (desc.type == BPF_MAP_TYPE_HASH ||
  114. desc.type == BPF_MAP_TYPE_LRU_HASH ||
  115. desc.type == BPF_MAP_TYPE_PERCPU_HASH ||
  116. desc.type == BPF_MAP_TYPE_HASH_OF_MAPS) {
  117. // For hash maps, use the first() interface (which uses get_next_key) to
  118. // iterate through the map and clear elements
  119. auto key = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.key_size),
  120. ::free);
  121. while (this->first(key.get()))
  122. if (!this->remove(key.get())) {
  123. return StatusTuple(-1,
  124. "Failed to delete element when clearing table %s",
  125. desc.name.c_str());
  126. }
  127. } else if (desc.type == BPF_MAP_TYPE_ARRAY ||
  128. desc.type == BPF_MAP_TYPE_PERCPU_ARRAY) {
  129. return StatusTuple(-1, "Array map %s do not support clearing elements",
  130. desc.name.c_str());
  131. } else if (desc.type == BPF_MAP_TYPE_PROG_ARRAY ||
  132. desc.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
  133. desc.type == BPF_MAP_TYPE_STACK_TRACE ||
  134. desc.type == BPF_MAP_TYPE_ARRAY_OF_MAPS) {
  135. // For Stack-trace and FD arrays, just iterate over all indices
  136. for (size_t i = 0; i < desc.max_entries; i++) {
  137. this->remove(&i);
  138. }
  139. } else {
  140. return StatusTuple(-1, "Clearing for map type of %s not supported yet",
  141. desc.name.c_str());
  142. }
  143. return StatusTuple::OK();
  144. }
  145. StatusTuple BPFTable::get_table_offline(
  146. std::vector<std::pair<std::string, std::string>> &res) {
  147. StatusTuple r(0);
  148. int err;
  149. auto key = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.key_size),
  150. ::free);
  151. auto value = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.leaf_size),
  152. ::free);
  153. std::string key_str;
  154. std::string value_str;
  155. if (desc.type == BPF_MAP_TYPE_ARRAY ||
  156. desc.type == BPF_MAP_TYPE_PROG_ARRAY ||
  157. desc.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
  158. desc.type == BPF_MAP_TYPE_PERCPU_ARRAY ||
  159. desc.type == BPF_MAP_TYPE_CGROUP_ARRAY ||
  160. desc.type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
  161. desc.type == BPF_MAP_TYPE_DEVMAP ||
  162. desc.type == BPF_MAP_TYPE_CPUMAP ||
  163. desc.type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
  164. // For arrays, just iterate over all indices
  165. for (size_t i = 0; i < desc.max_entries; i++) {
  166. err = bpf_lookup_elem(desc.fd, &i, value.get());
  167. if (err < 0 && errno == ENOENT) {
  168. // Element is not present, skip it
  169. continue;
  170. } else if (err < 0) {
  171. // Other error, abort
  172. return StatusTuple(-1, "Error looking up value: %s", std::strerror(errno));
  173. }
  174. r = key_to_string(&i, key_str);
  175. if (!r.ok())
  176. return r;
  177. r = leaf_to_string(value.get(), value_str);
  178. if (!r.ok())
  179. return r;
  180. res.emplace_back(key_str, value_str);
  181. }
  182. } else {
  183. res.clear();
  184. // For other maps, try to use the first() and next() interfaces
  185. if (!this->first(key.get()))
  186. return StatusTuple::OK();
  187. while (true) {
  188. if (!this->lookup(key.get(), value.get()))
  189. break;
  190. r = key_to_string(key.get(), key_str);
  191. if (!r.ok())
  192. return r;
  193. r = leaf_to_string(value.get(), value_str);
  194. if (!r.ok())
  195. return r;
  196. res.emplace_back(key_str, value_str);
  197. if (!this->next(key.get(), key.get()))
  198. break;
  199. }
  200. }
  201. return StatusTuple::OK();
  202. }
  203. StatusTuple BPFTable::get_table_offline_ptr(
  204. std::vector<std::pair<std::vector<char>, std::vector<char>>> &res) {
  205. StatusTuple r(0);
  206. int err;
  207. auto key = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.key_size),
  208. ::free);
  209. auto value = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.leaf_size),
  210. ::free);
  211. std::string key_str;
  212. std::string value_str;
  213. if (desc.type == BPF_MAP_TYPE_ARRAY ||
  214. desc.type == BPF_MAP_TYPE_PROG_ARRAY ||
  215. desc.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
  216. desc.type == BPF_MAP_TYPE_PERCPU_ARRAY ||
  217. desc.type == BPF_MAP_TYPE_CGROUP_ARRAY ||
  218. desc.type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
  219. desc.type == BPF_MAP_TYPE_DEVMAP ||
  220. desc.type == BPF_MAP_TYPE_CPUMAP ||
  221. desc.type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
  222. // For arrays, just iterate over all indices
  223. for (size_t i = 0; i < desc.max_entries; i++) {
  224. err = bpf_lookup_elem(desc.fd, &i, value.get());
  225. if (err < 0 && errno == ENOENT) {
  226. // Element is not present, skip it
  227. continue;
  228. } else if (err < 0) {
  229. // Other error, abort
  230. return StatusTuple(-1, "Error looking up value: %s", std::strerror(errno));
  231. }
  232. r = key_to_string(&i, key_str);
  233. if (!r.ok())
  234. return r;
  235. r = leaf_to_string(value.get(), value_str);
  236. if (!r.ok())
  237. return r;
  238. std::vector<char> key_vec(desc.key_size);
  239. std::vector<char> value_vec(desc.leaf_size);
  240. std::memcpy(key_vec.data(), key.get(), desc.key_size);
  241. std::memcpy(value_vec.data(), value.get(), desc.leaf_size);
  242. res.emplace_back(key_vec, value_vec);
  243. }
  244. } else {
  245. res.clear();
  246. // For other maps, try to use the first() and next() interfaces
  247. if (!this->first(key.get()))
  248. return StatusTuple::OK();
  249. while (true) {
  250. if (!this->lookup(key.get(), value.get()))
  251. break;
  252. r = key_to_string(key.get(), key_str);
  253. if (!r.ok())
  254. return r;
  255. r = leaf_to_string(value.get(), value_str);
  256. if (!r.ok())
  257. return r;
  258. std::vector<char> key_vec(desc.key_size);
  259. std::vector<char> value_vec(desc.leaf_size);
  260. std::memcpy(key_vec.data(), key.get(), desc.key_size);
  261. std::memcpy(value_vec.data(), value.get(), desc.leaf_size);
  262. res.emplace_back(key_vec, value_vec);
  263. if (!this->next(key.get(), key.get()))
  264. break;
  265. }
  266. }
  267. return StatusTuple::OK();
  268. }
  269. size_t BPFTable::get_possible_cpu_count() { return get_possible_cpus().size(); }
  270. BPFStackTable::BPFStackTable(const TableDesc& desc, bool use_debug_file,
  271. bool check_debug_file_crc)
  272. : BPFTableBase<int, stacktrace_t>(desc) {
  273. if (desc.type != BPF_MAP_TYPE_STACK_TRACE)
  274. throw std::invalid_argument("Table '" + desc.name +
  275. "' is not a stack table");
  276. uint32_t use_symbol_type = (1 << STT_FUNC) | (1 << STT_GNU_IFUNC);
  277. symbol_option_ = {.use_debug_file = use_debug_file,
  278. .check_debug_file_crc = check_debug_file_crc,
  279. .lazy_symbolize = 1,
  280. .use_symbol_type = use_symbol_type};
  281. }
  282. BPFStackTable::BPFStackTable(BPFStackTable&& that)
  283. : BPFTableBase<int, stacktrace_t>(that.desc),
  284. symbol_option_(std::move(that.symbol_option_)),
  285. pid_sym_(std::move(that.pid_sym_)) {
  286. that.pid_sym_.clear();
  287. }
  288. BPFStackTable::~BPFStackTable() {
  289. for (auto it : pid_sym_)
  290. bcc_free_symcache(it.second, it.first);
  291. }
  292. void BPFStackTable::free_symcache(int pid) {
  293. auto iter = pid_sym_.find(pid);
  294. if (iter != pid_sym_.end()) {
  295. bcc_free_symcache(iter->second, iter->first);
  296. pid_sym_.erase(iter);
  297. }
  298. }
  299. void BPFStackTable::clear_table_non_atomic() {
  300. for (int i = 0; size_t(i) < capacity(); i++) {
  301. remove(&i);
  302. }
  303. }
  304. std::vector<uintptr_t> BPFStackTable::get_stack_addr(int stack_id) {
  305. std::vector<uintptr_t> res;
  306. stacktrace_t stack;
  307. if (stack_id < 0)
  308. return res;
  309. if (!lookup(&stack_id, &stack))
  310. return res;
  311. for (int i = 0; (i < BPF_MAX_STACK_DEPTH) && (stack.ip[i] != 0); i++)
  312. res.push_back(stack.ip[i]);
  313. return res;
  314. }
  315. std::vector<std::string> BPFStackTable::get_stack_symbol(int stack_id,
  316. int pid) {
  317. auto addresses = get_stack_addr(stack_id);
  318. std::vector<std::string> res;
  319. if (addresses.empty())
  320. return res;
  321. res.reserve(addresses.size());
  322. if (pid < 0)
  323. pid = -1;
  324. if (pid_sym_.find(pid) == pid_sym_.end())
  325. pid_sym_[pid] = bcc_symcache_new(pid, &symbol_option_);
  326. void* cache = pid_sym_[pid];
  327. bcc_symbol symbol;
  328. for (auto addr : addresses)
  329. if (bcc_symcache_resolve(cache, addr, &symbol) != 0)
  330. res.emplace_back("[UNKNOWN]");
  331. else {
  332. res.push_back(symbol.demangle_name);
  333. bcc_symbol_free_demangle_name(&symbol);
  334. }
  335. return res;
  336. }
  337. BPFStackBuildIdTable::BPFStackBuildIdTable(const TableDesc& desc, bool use_debug_file,
  338. bool check_debug_file_crc,
  339. void *bsymcache)
  340. : BPFTableBase<int, stacktrace_buildid_t>(desc),
  341. bsymcache_(bsymcache) {
  342. if (desc.type != BPF_MAP_TYPE_STACK_TRACE)
  343. throw std::invalid_argument("Table '" + desc.name +
  344. "' is not a stack table");
  345. symbol_option_ = {.use_debug_file = use_debug_file,
  346. .check_debug_file_crc = check_debug_file_crc,
  347. .lazy_symbolize = 1,
  348. .use_symbol_type = (1 << STT_FUNC) | (1 << STT_GNU_IFUNC)};
  349. }
  350. void BPFStackBuildIdTable::clear_table_non_atomic() {
  351. for (int i = 0; size_t(i) < capacity(); i++) {
  352. remove(&i);
  353. }
  354. }
  355. std::vector<bpf_stack_build_id> BPFStackBuildIdTable::get_stack_addr(int stack_id) {
  356. std::vector<bpf_stack_build_id> res;
  357. struct stacktrace_buildid_t stack;
  358. if (stack_id < 0)
  359. return res;
  360. if (!lookup(&stack_id, &stack))
  361. return res;
  362. for (int i = 0; (i < BPF_MAX_STACK_DEPTH) && \
  363. (stack.trace[i].status == BPF_STACK_BUILD_ID_VALID);
  364. i++) {
  365. /* End of stack marker is BCC_STACK_BUILD_ID_EMPTY or
  366. * BCC_STACK_BUILD_IP(fallback) mechanism.
  367. * We do not support fallback mechanism
  368. */
  369. res.push_back(stack.trace[i]);
  370. }
  371. return res;
  372. }
  373. std::vector<std::string> BPFStackBuildIdTable::get_stack_symbol(int stack_id)
  374. {
  375. auto addresses = get_stack_addr(stack_id);
  376. std::vector<std::string> res;
  377. if (addresses.empty())
  378. return res;
  379. res.reserve(addresses.size());
  380. bcc_symbol symbol;
  381. struct bpf_stack_build_id trace;
  382. for (auto addr : addresses) {
  383. memcpy(trace.build_id, addr.build_id, sizeof(trace.build_id));
  384. trace.status = addr.status;
  385. trace.offset = addr.offset;
  386. if (bcc_buildsymcache_resolve(bsymcache_,&trace,&symbol) != 0) {
  387. res.emplace_back("[UNKNOWN]");
  388. } else {
  389. res.push_back(symbol.name);
  390. bcc_symbol_free_demangle_name(&symbol);
  391. }
  392. }
  393. return res;
  394. }
  395. BPFPerfBuffer::BPFPerfBuffer(const TableDesc& desc)
  396. : BPFTableBase<int, int>(desc), epfd_(-1) {
  397. if (desc.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
  398. throw std::invalid_argument("Table '" + desc.name +
  399. "' is not a perf buffer");
  400. }
  401. StatusTuple BPFPerfBuffer::open_on_cpu(perf_reader_raw_cb cb, perf_reader_lost_cb lost_cb,
  402. void* cb_cookie, int page_cnt,
  403. struct bcc_perf_buffer_opts& opts) {
  404. if (cpu_readers_.find(opts.cpu) != cpu_readers_.end())
  405. return StatusTuple(-1, "Perf buffer already open on CPU %d", opts.cpu);
  406. auto reader = static_cast<perf_reader*>(
  407. bpf_open_perf_buffer_opts(cb, lost_cb, cb_cookie, page_cnt, &opts));
  408. if (reader == nullptr)
  409. return StatusTuple(-1, "Unable to construct perf reader");
  410. int reader_fd = perf_reader_fd(reader);
  411. if (!update(&opts.cpu, &reader_fd)) {
  412. perf_reader_free(static_cast<void*>(reader));
  413. return StatusTuple(-1, "Unable to open perf buffer on CPU %d: %s", opts.cpu,
  414. std::strerror(errno));
  415. }
  416. struct epoll_event event = {};
  417. event.events = EPOLLIN;
  418. event.data.ptr = static_cast<void*>(reader);
  419. if (epoll_ctl(epfd_, EPOLL_CTL_ADD, reader_fd, &event) != 0) {
  420. perf_reader_free(static_cast<void*>(reader));
  421. return StatusTuple(-1, "Unable to add perf_reader FD to epoll: %s",
  422. std::strerror(errno));
  423. }
  424. cpu_readers_[opts.cpu] = reader;
  425. return StatusTuple::OK();
  426. }
  427. StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb,
  428. perf_reader_lost_cb lost_cb,
  429. void* cb_cookie, int page_cnt) {
  430. return open_all_cpu(cb, lost_cb, cb_cookie, page_cnt, 1);
  431. }
  432. StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb,
  433. perf_reader_lost_cb lost_cb,
  434. void* cb_cookie, int page_cnt,
  435. int wakeup_events)
  436. {
  437. if (cpu_readers_.size() != 0 || epfd_ != -1)
  438. return StatusTuple(-1, "Previously opened perf buffer not cleaned");
  439. std::vector<int> cpus = get_online_cpus();
  440. ep_events_.reset(new epoll_event[cpus.size()]);
  441. epfd_ = epoll_create1(EPOLL_CLOEXEC);
  442. for (int i : cpus) {
  443. struct bcc_perf_buffer_opts opts = {
  444. .pid = -1,
  445. .cpu = i,
  446. .wakeup_events = wakeup_events,
  447. };
  448. auto res = open_on_cpu(cb, lost_cb, cb_cookie, page_cnt, opts);
  449. if (!res.ok()) {
  450. TRY2(close_all_cpu());
  451. return res;
  452. }
  453. }
  454. return StatusTuple::OK();
  455. }
  456. StatusTuple BPFPerfBuffer::close_on_cpu(int cpu) {
  457. auto it = cpu_readers_.find(cpu);
  458. if (it == cpu_readers_.end())
  459. return StatusTuple::OK();
  460. perf_reader_free(static_cast<void*>(it->second));
  461. if (!remove(const_cast<int*>(&(it->first))))
  462. return StatusTuple(-1, "Unable to close perf buffer on CPU %d", it->first);
  463. cpu_readers_.erase(it);
  464. return StatusTuple::OK();
  465. }
  466. StatusTuple BPFPerfBuffer::close_all_cpu() {
  467. std::string errors;
  468. bool has_error = false;
  469. if (epfd_ >= 0) {
  470. int close_res = close(epfd_);
  471. epfd_ = -1;
  472. ep_events_.reset();
  473. if (close_res != 0) {
  474. has_error = true;
  475. errors += std::string(std::strerror(errno)) + "\n";
  476. }
  477. }
  478. std::vector<int> opened_cpus;
  479. for (auto it : cpu_readers_)
  480. opened_cpus.push_back(it.first);
  481. for (int i : opened_cpus) {
  482. auto res = close_on_cpu(i);
  483. if (!res.ok()) {
  484. errors += "Failed to close CPU" + std::to_string(i) + " perf buffer: ";
  485. errors += res.msg() + "\n";
  486. has_error = true;
  487. }
  488. }
  489. if (has_error)
  490. return StatusTuple(-1, errors);
  491. return StatusTuple::OK();
  492. }
  493. int BPFPerfBuffer::poll(int timeout_ms) {
  494. if (epfd_ < 0)
  495. return -1;
  496. int cnt =
  497. epoll_wait(epfd_, ep_events_.get(), cpu_readers_.size(), timeout_ms);
  498. for (int i = 0; i < cnt; i++)
  499. perf_reader_event_read(static_cast<perf_reader*>(ep_events_[i].data.ptr));
  500. return cnt;
  501. }
  502. int BPFPerfBuffer::consume() {
  503. if (epfd_ < 0)
  504. return -1;
  505. for (auto it : cpu_readers_)
  506. perf_reader_event_read(it.second);
  507. return 0;
  508. }
  509. BPFPerfBuffer::~BPFPerfBuffer() {
  510. auto res = close_all_cpu();
  511. if (!res.ok())
  512. std::cerr << "Failed to close all perf buffer on destruction: " << res.msg()
  513. << std::endl;
  514. }
  515. BPFPerfEventArray::BPFPerfEventArray(const TableDesc& desc)
  516. : BPFTableBase<int, int>(desc) {
  517. if (desc.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
  518. throw std::invalid_argument("Table '" + desc.name +
  519. "' is not a perf event array");
  520. }
  521. StatusTuple BPFPerfEventArray::open_all_cpu(uint32_t type, uint64_t config,
  522. int pid) {
  523. if (cpu_fds_.size() != 0)
  524. return StatusTuple(-1, "Previously opened perf event not cleaned");
  525. std::vector<int> cpus = get_online_cpus();
  526. for (int i : cpus) {
  527. auto res = open_on_cpu(i, type, config, pid);
  528. if (!res.ok()) {
  529. TRY2(close_all_cpu());
  530. return res;
  531. }
  532. }
  533. return StatusTuple::OK();
  534. }
  535. StatusTuple BPFPerfEventArray::close_all_cpu() {
  536. std::string errors;
  537. bool has_error = false;
  538. std::vector<int> opened_cpus;
  539. for (auto it : cpu_fds_)
  540. opened_cpus.push_back(it.first);
  541. for (int i : opened_cpus) {
  542. auto res = close_on_cpu(i);
  543. if (!res.ok()) {
  544. errors += "Failed to close CPU" + std::to_string(i) + " perf event: ";
  545. errors += res.msg() + "\n";
  546. has_error = true;
  547. }
  548. }
  549. if (has_error)
  550. return StatusTuple(-1, errors);
  551. return StatusTuple::OK();
  552. }
  553. StatusTuple BPFPerfEventArray::open_on_cpu(int cpu, uint32_t type,
  554. uint64_t config, int pid) {
  555. if (cpu_fds_.find(cpu) != cpu_fds_.end())
  556. return StatusTuple(-1, "Perf event already open on CPU %d", cpu);
  557. int fd = bpf_open_perf_event(type, config, pid, cpu);
  558. if (fd < 0) {
  559. return StatusTuple(-1, "Error constructing perf event %" PRIu32 ":%" PRIu64,
  560. type, config);
  561. }
  562. if (!update(&cpu, &fd)) {
  563. bpf_close_perf_event_fd(fd);
  564. return StatusTuple(-1, "Unable to open perf event on CPU %d: %s", cpu,
  565. std::strerror(errno));
  566. }
  567. cpu_fds_[cpu] = fd;
  568. return StatusTuple::OK();
  569. }
  570. StatusTuple BPFPerfEventArray::close_on_cpu(int cpu) {
  571. auto it = cpu_fds_.find(cpu);
  572. if (it == cpu_fds_.end()) {
  573. return StatusTuple::OK();
  574. }
  575. bpf_close_perf_event_fd(it->second);
  576. cpu_fds_.erase(it);
  577. return StatusTuple::OK();
  578. }
  579. BPFPerfEventArray::~BPFPerfEventArray() {
  580. auto res = close_all_cpu();
  581. if (!res.ok()) {
  582. std::cerr << "Failed to close all perf buffer on destruction: " << res.msg()
  583. << std::endl;
  584. }
  585. }
  586. BPFProgTable::BPFProgTable(const TableDesc& desc)
  587. : BPFTableBase<int, int>(desc) {
  588. if (desc.type != BPF_MAP_TYPE_PROG_ARRAY)
  589. throw std::invalid_argument("Table '" + desc.name +
  590. "' is not a prog table");
  591. }
  592. StatusTuple BPFProgTable::update_value(const int& index, const int& prog_fd) {
  593. if (!this->update(const_cast<int*>(&index), const_cast<int*>(&prog_fd)))
  594. return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  595. return StatusTuple::OK();
  596. }
  597. StatusTuple BPFProgTable::remove_value(const int& index) {
  598. if (!this->remove(const_cast<int*>(&index)))
  599. return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  600. return StatusTuple::OK();
  601. }
  602. BPFCgroupArray::BPFCgroupArray(const TableDesc& desc)
  603. : BPFTableBase<int, int>(desc) {
  604. if (desc.type != BPF_MAP_TYPE_CGROUP_ARRAY)
  605. throw std::invalid_argument("Table '" + desc.name +
  606. "' is not a cgroup array");
  607. }
  608. StatusTuple BPFCgroupArray::update_value(const int& index,
  609. const int& cgroup2_fd) {
  610. if (!this->update(const_cast<int*>(&index), const_cast<int*>(&cgroup2_fd)))
  611. return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  612. return StatusTuple::OK();
  613. }
  614. StatusTuple BPFCgroupArray::update_value(const int& index,
  615. const std::string& cgroup2_path) {
  616. FileDesc f(::open(cgroup2_path.c_str(), O_RDONLY | O_CLOEXEC));
  617. if ((int)f < 0)
  618. return StatusTuple(-1, "Unable to open %s", cgroup2_path.c_str());
  619. TRY2(update_value(index, (int)f));
  620. return StatusTuple::OK();
  621. }
  622. StatusTuple BPFCgroupArray::remove_value(const int& index) {
  623. if (!this->remove(const_cast<int*>(&index)))
  624. return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  625. return StatusTuple::OK();
  626. }
  627. BPFDevmapTable::BPFDevmapTable(const TableDesc& desc)
  628. : BPFTableBase<int, int>(desc) {
  629. if(desc.type != BPF_MAP_TYPE_DEVMAP)
  630. throw std::invalid_argument("Table '" + desc.name +
  631. "' is not a devmap table");
  632. }
  633. StatusTuple BPFDevmapTable::update_value(const int& index,
  634. const int& value) {
  635. if (!this->update(const_cast<int*>(&index), const_cast<int*>(&value)))
  636. return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  637. return StatusTuple::OK();
  638. }
  639. StatusTuple BPFDevmapTable::get_value(const int& index,
  640. int& value) {
  641. if (!this->lookup(const_cast<int*>(&index), &value))
  642. return StatusTuple(-1, "Error getting value: %s", std::strerror(errno));
  643. return StatusTuple::OK();
  644. }
  645. StatusTuple BPFDevmapTable::remove_value(const int& index) {
  646. if (!this->remove(const_cast<int*>(&index)))
  647. return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  648. return StatusTuple::OK();
  649. }
  650. BPFXskmapTable::BPFXskmapTable(const TableDesc& desc)
  651. : BPFTableBase<int, int>(desc) {
  652. if(desc.type != BPF_MAP_TYPE_XSKMAP)
  653. throw std::invalid_argument("Table '" + desc.name +
  654. "' is not a xskmap table");
  655. }
  656. StatusTuple BPFXskmapTable::update_value(const int& index,
  657. const int& value) {
  658. if (!this->update(const_cast<int*>(&index), const_cast<int*>(&value)))
  659. return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  660. return StatusTuple::OK();
  661. }
  662. StatusTuple BPFXskmapTable::get_value(const int& index,
  663. int& value) {
  664. if (!this->lookup(const_cast<int*>(&index), &value))
  665. return StatusTuple(-1, "Error getting value: %s", std::strerror(errno));
  666. return StatusTuple::OK();
  667. }
  668. StatusTuple BPFXskmapTable::remove_value(const int& index) {
  669. if (!this->remove(const_cast<int*>(&index)))
  670. return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  671. return StatusTuple::OK();
  672. }
  673. BPFSockmapTable::BPFSockmapTable(const TableDesc& desc)
  674. : BPFTableBase<int, int>(desc) {
  675. if(desc.type != BPF_MAP_TYPE_SOCKMAP)
  676. throw std::invalid_argument("Table '" + desc.name +
  677. "' is not a sockmap table");
  678. }
  679. StatusTuple BPFSockmapTable::update_value(const int& index,
  680. const int& value) {
  681. if (!this->update(const_cast<int*>(&index), const_cast<int*>(&value)))
  682. return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  683. return StatusTuple::OK();
  684. }
  685. StatusTuple BPFSockmapTable::remove_value(const int& index) {
  686. if (!this->remove(const_cast<int*>(&index)))
  687. return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  688. return StatusTuple::OK();
  689. }
  690. BPFSockhashTable::BPFSockhashTable(const TableDesc& desc)
  691. : BPFTableBase<int, int>(desc) {
  692. if(desc.type != BPF_MAP_TYPE_SOCKHASH)
  693. throw std::invalid_argument("Table '" + desc.name +
  694. "' is not a sockhash table");
  695. }
  696. StatusTuple BPFSockhashTable::update_value(const int& key,
  697. const int& value) {
  698. if (!this->update(const_cast<int*>(&key), const_cast<int*>(&value)))
  699. return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  700. return StatusTuple::OK();
  701. }
  702. StatusTuple BPFSockhashTable::remove_value(const int& key) {
  703. if (!this->remove(const_cast<int*>(&key)))
  704. return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  705. return StatusTuple::OK();
  706. }
  707. } // namespace ebpf