Cheri-research/FAT-Allocator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all commits for a clean git history

Browse Source
This commit is contained in:
Akilan Selvacoumar
2024-09-18 14:22:27 +01:00
commit 18306c66fc
6128 changed files with 3186840 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
.DS_Store vendored Normal file
View File

Binary file not shown.

5
.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
# ignore shared object files
*.so
# ignore .out binaries
*.out

8
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

8
.idea/Allocator.iml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="CPP_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

8
.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/Allocator.iml" filepath="$PROJECT_DIR$/.idea/Allocator.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

19
.vscode/c_cpp_properties.json vendored Normal file
View File

@@ -0,0 +1,19 @@
{
"configurations": [
{
"name": "Mac",
"includePath": [
"${workspaceFolder}/**"
],
"defines": [],
"macFrameworkPath": [
"/Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/System/Library/Frameworks"
],
"compilerPath": "/opt/homebrew/opt/llvm/bin/clang",
"cStandard": "c17",
"cppStandard": "c++17",
"intelliSenseMode": "macos-clang-arm64"
}
],
"version": 4
}

40
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,40 @@
{
"files.associations": {
"eal_internal_cfg.h": "c",
"rte_eal.h": "c",
"rte_compat.h": "c",
"rte_pci_dev_feature_defs.h": "c",
"rte_config.h": "c",
"eal_hugepages.h": "c",
"eal_filesystem.h": "c",
"memory_resource": "c",
"iterator": "c",
"locale": "c",
"__config": "c",
"vector": "c",
"rte_pause.h": "c",
"rte_common.h": "c",
"__hash_table": "c",
"__split_buffer": "c",
"__tree": "c",
"array": "c",
"bitset": "c",
"deque": "c",
"initializer_list": "c",
"map": "c",
"queue": "c",
"set": "c",
"unordered_map": "c",
"unordered_set": "c",
"string": "c",
"string_view": "c",
"stddefines.h": "c",
"charconv": "c",
"coz.h": "c",
"assert.h": "c",
"mman.h": "c",
"time.h": "c",
"vmmeter.h": "c"
},
"C_Cpp.errorSquiggles": "disabled"
}

89
OverwriteMalloc.h Normal file
View File

@@ -0,0 +1,89 @@
//
// Created by Akilan on 07/03/2024.
//
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <dlfcn.h>
#include <sys/fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/wait.h>
#include <assert.h>
#include <sys/time.h>
#include <stdint.h>
#include <stdio.h>
#define FILENAME "/dev/contigmem"
static inline void *MALLOC(size_t size)
{
// void * temp = malloc(size);
// assert(temp);
// return temp;
// Ensuring malloc this is mmap and phyiscally contigous
// void *ptr = malloc(size);
// return ptr;
}
// Currently not implementing free
// This is to ensure the PureCap
// program can terminate itself
// even though there is memory not
// cleared.
static inline void *FREE(void *ptr)
{
// int *pt = ptr;
// size_t size;
// --pt;
// size = *pt;
// if(munmap(pt, size) == -1)
// {
// perror("munmap");
// exit(EXIT_FAILURE);
// }
// free(ptr);
}
inline void *MALLOCPHYSICALLCONTIGOUS(size_t size) {
int fd = open
(FILENAME, O_RDWR | O_CREAT, 0600);
if (fd < 0) {
perror("open");
exit(EXIT_FAILURE);
}
off_t offset = 0; // offset to seek to.
if (ftruncate(fd, size) < 0) {
perror("ftruncate");
close(fd);
exit(EXIT_FAILURE);
}
int *ptr = mmap(NULL, size,
PROT_READ | PROT_WRITE, MAP_SHARED,
fd, offset);
if(ptr == MAP_FAILED)
{
perror("mmap");
exit(EXIT_FAILURE);
}
*ptr = size;
return ptr;
}

388
Programs/Kmeans.c Normal file
View File

@@ -0,0 +1,388 @@
/* Copyright (c) 2007-2009, Stanford University
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Stanford University nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY STANFORD UNIVERSITY ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL STANFORD UNIVERSITY BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <strings.h>
#include <string.h>
#include <stddef.h>
#include <stdlib.h>
#include <unistd.h>
#include <assert.h>
#include <string.h>
#include <math.h>
#include <pthread.h>
#include "stddefines.h"
#include "coz.h"
#define malloc MALLOCCONTIGOUS
#define free FREE
// #include "prof.h"
// default settings
// #define DEF_NUM_POINTS 100000
// #define DEF_NUM_MEANS 100
// #define DEF_DIM 3
// #define DEF_GRID_SIZE 1000
// setting smaller parameters
// to detect root cause
#define DEF_NUM_POINTS 1000000
#define DEF_NUM_MEANS 10
#define DEF_DIM 3
#define DEF_GRID_SIZE 10
#define false 0
#define true 1
int num_points; // number of vectors
int dim; // Dimension of each vector
int num_means; // number of clusters
int grid_size; // size of each dimension of vector space
int modified;
int num_pts = 0;
int **points;
int **means;
int *clusters;
typedef struct {
int start_idx;
int num_pts;
int *sum;
} thread_arg;
/** dump_points()
* Helper function to print out the points
*/
void dump_points(int **vals, int rows)
{
int i, j;
for (i = 0; i < rows; i++)
{
for (j = 0; j < dim; j++)
{
dprintf("%5d ",vals[i][j]);
}
dprintf("\n");
}
}
/** parse_args()
* Parse the user arguments
*/
void parse_args(int argc, char **argv)
{
int c;
extern char *optarg;
extern int optind;
num_points = DEF_NUM_POINTS;
num_means = DEF_NUM_MEANS;
dim = DEF_DIM;
grid_size = DEF_GRID_SIZE;
while ((c = getopt(argc, argv, "d:c:p:s:")) != EOF)
{
switch (c) {
case 'd':
dim = atoi(optarg);
break;
case 'c':
num_means = atoi(optarg);
break;
case 'p':
num_points = atoi(optarg);
break;
case 's':
grid_size = atoi(optarg);
break;
case '?':
printf("Usage: %s -d <vector dimension> -c <num clusters> -p <num points> -s <grid size>\n", argv[0]);
exit(1);
}
}
if (dim <= 0 || num_means <= 0 || num_points <= 0 || grid_size <= 0) {
printf("Illegal argument value. All values must be numeric and greater than 0\n");
exit(1);
}
printf("Dimension = %d\n", dim);
printf("Number of clusters = %d\n", num_means);
printf("Number of points = %d\n", num_points);
printf("Size of each dimension = %d\n", grid_size);
}
/** generate_points()
* Generate the points
*/
void generate_points(int **pts, int size)
{
dprintf("outer loop\n");
int i, j;
for (i=0; i<size; i++)
{
for (j=0; j<dim; j++)
{
dprintf("inner loop\n");
// dprintf(&pts[i][j]);
pts[i][j] = rand() % grid_size;
}
}
}
/** get_sq_dist()
* Get the squared distance between 2 points
*/
static inline unsigned int get_sq_dist(int *v1, int *v2)
{
int i;
unsigned int sum = 0;
for (i = 0; i < dim; i++)
{
sum += ((v1[i] - v2[i]) * (v1[i] - v2[i]));
}
return sum;
}
/** add_to_sum()
* Helper function to update the total distance sum
*/
void add_to_sum(int *sum, int *point)
{
int i;
for (i = 0; i < dim; i++)
{
sum[i] += point[i];
}
}
/** find_clusters()
* Find the cluster that is most suitable for a given set of points
*/
void *find_clusters(void *arg)
{
thread_arg *t_arg = (thread_arg *)arg;
int i, j;
unsigned int min_dist, cur_dist;
int min_idx;
int start_idx = t_arg->start_idx;
int end_idx = start_idx + t_arg->num_pts;
for (i = start_idx; i < end_idx; i++)
{
min_dist = get_sq_dist(points[i], means[0]);
min_idx = 0;
for (j = 1; j < num_means; j++)
{
cur_dist = get_sq_dist(points[i], means[j]);
if (cur_dist < min_dist)
{
min_dist = cur_dist;
min_idx = j;
}
}
if (clusters[i] != min_idx)
{
clusters[i] = min_idx;
modified = true;
}
// COZ_PROGRESS_NAMED("clusters found");
}
return (void *)0;
}
/** calc_means()
* Compute the means for the various clusters
*/
void *calc_means(void *arg)
{
int i, j, grp_size;
int *sum;
thread_arg *t_arg = (thread_arg *)arg;
int start_idx = t_arg->start_idx;
int end_idx = start_idx + t_arg->num_pts;
sum = t_arg->sum;
for (i = start_idx; i < end_idx; i++)
{
memset(sum, 0, dim * sizeof(int));
grp_size = 0;
for (j = 0; j < num_points; j++)
{
if (clusters[j] == i)
{
add_to_sum(sum, points[j]);
grp_size++;
}
}
for (j = 0; j < dim; j++)
{
//dprintf("div sum = %d, grp size = %d\n", sum[j], grp_size);
if (grp_size != 0)
{
means[i][j] = sum[j] / grp_size;
}
}
}
free(sum);
return (void *)0;
}
int main(int argc, char **argv)
{
int num_procs, curr_point;
int i;
pthread_t *pid;
pthread_attr_t attr;
thread_arg *arg;
int num_per_thread, excess;
parse_args(argc, argv);
// PROF_START();
points = (int **)malloc(sizeof(int *) * num_points);
for (i=0; i<num_points; i++)
{
points[i] = (int *)malloc(sizeof(int) * dim);
}
dprintf("Generating points\n");
dprintf("here\n");
// generate_points(points, num_points);
// means = (int **)malloc(sizeof(int *) * num_means);
// for (i=0; i<num_means; i++)
// {
// means[i] = (int *)malloc(sizeof(int) * dim);
// }
// dprintf("Generating means\n");
// generate_points(means, num_means);
// clusters = (int *)malloc(sizeof(int) * num_points);
// memset(clusters, -1, sizeof(int) * num_points);
// pthread_attr_init(&attr);
// pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
// CHECK_ERROR((num_procs = sysconf(_SC_NPROCESSORS_ONLN)) <= 0);
// CHECK_ERROR( (pid = (pthread_t *)malloc(sizeof(pthread_t) * num_procs)) == NULL);
// modified = true;
// printf("Starting iterative algorithm\n");
// /* Create the threads to process the distances between the various
// points and repeat until modified is no longer valid */
// int num_threads;
// while (modified)
// {
// num_per_thread = num_points / num_procs;
// excess = num_points % num_procs;
// modified = false;
// dprintf(".");
// curr_point = 0;
// num_threads = 0;
// while (curr_point < num_points) {
// CHECK_ERROR((arg = (thread_arg *)malloc(sizeof(thread_arg))) == NULL);
// arg->start_idx = curr_point;
// arg->num_pts = num_per_thread;
// if (excess > 0) {
// arg->num_pts++;
// excess--;
// }
// CHECK_ERROR((pthread_create(&(pid[num_threads++]), &attr, find_clusters,
// (void *)(arg))) != 0);
// curr_point += arg->num_pts;
// }
// assert (num_threads == num_procs);
// for (i = 0; i < num_threads; i++) {
// pthread_join(pid[i], NULL);
// }
// num_per_thread = num_means / num_procs;
// excess = num_means % num_procs;
// curr_point = 0;
// num_threads = 0;
// while (curr_point < num_means) {
// CHECK_ERROR((arg = (thread_arg *)malloc(sizeof(thread_arg))) == NULL);
// arg->start_idx = curr_point;
// arg->sum = (int *)malloc(dim * sizeof(int));
// arg->num_pts = num_per_thread;
// if (excess > 0) {
// arg->num_pts++;
// excess--;
// }
// CHECK_ERROR((pthread_create(&(pid[num_threads++]), &attr, calc_means,
// (void *)(arg))) != 0);
// curr_point += arg->num_pts;
// }
// assert (num_threads == num_procs);
// for (i = 0; i < num_threads; i++) {
// pthread_join(pid[i], NULL);
// }
// }
// dprintf("\n\nFinal means:\n");
// dump_points(means, num_means);
// for (i = 0; i < num_points; i++)
// free(points[i]);
// free(points);
// for (i = 0; i < num_means; i++)
// {
// free(means[i]);
// }
// free(means);
// free(clusters);
// // PROF_STDOUT();
return 0;
}

1
Programs/build.sh Normal file
View File

@@ -0,0 +1 @@
cc -g -Wall -o Kmeans.out -mabi=purecap-benchmark -lpthread Kmeans.c

93
Programs/coz.h Normal file
View File

@@ -0,0 +1,93 @@
/*
* Copyright (c) 2015, Charlie Curtsinger and Emery Berger,
* University of Massachusetts Amherst
* This file is part of the Coz project. See LICENSE.md file at the top-level
* directory of this distribution and at http://github.com/plasma-umass/coz.
*/
#if !defined(COZ_H)
#define COZ_H
#ifndef __USE_GNU
# define __USE_GNU
#endif
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include <dlfcn.h>
#include <stdint.h>
#include <string.h> /* for memcpy hack below */
#if defined(__cplusplus)
extern "C" {
#endif
#define COZ_COUNTER_TYPE_THROUGHPUT 1
#define COZ_COUNTER_TYPE_BEGIN 2
#define COZ_COUNTER_TYPE_END 3
// Declare dlsym as a weak reference so libdl isn't required
void* dlsym(void* handle, const char* symbol) __attribute__((weak));
// Counter info struct, containing both a counter and backoff size
typedef struct {
size_t count; // The actual count
size_t backoff; // Used to batch updates to the shared counter. Currently unused.
} coz_counter_t;
// The type of the _coz_get_counter function
typedef coz_counter_t* (*coz_get_counter_t)(int, const char*);
// Locate and invoke _coz_get_counter
static coz_counter_t* _call_coz_get_counter(int type, const char* name) {
static unsigned char _initialized = 0;
static coz_get_counter_t fn; // The pointer to _coz_get_counter
if(!_initialized) {
if(dlsym) {
// Locate the _coz_get_counter method
void* p = dlsym(RTLD_DEFAULT, "_coz_get_counter");
// Use memcpy to avoid pedantic GCC complaint about storing function pointer in void*
memcpy(&fn, &p, sizeof(p));
}
_initialized = 1;
}
// Call the function, or return null if profiler is not found
if(fn) return fn(type, name);
else return 0;
}
// Macro to initialize and increment a counter
#define COZ_INCREMENT_COUNTER(type, name) \
if(1) { \
static unsigned char _initialized = 0; \
static coz_counter_t* _counter = 0; \
\
if(!_initialized) { \
_counter = _call_coz_get_counter(type, name); \
_initialized = 1; \
} \
if(_counter) { \
__atomic_add_fetch(&_counter->count, 1, __ATOMIC_RELAXED); \
} \
}
#define STR2(x) #x
#define STR(x) STR2(x)
#define COZ_PROGRESS_NAMED(name) COZ_INCREMENT_COUNTER(COZ_COUNTER_TYPE_THROUGHPUT, name)
#define COZ_PROGRESS COZ_INCREMENT_COUNTER(COZ_COUNTER_TYPE_THROUGHPUT, __FILE__ ":" STR(__LINE__))
#define COZ_BEGIN(name) COZ_INCREMENT_COUNTER(COZ_COUNTER_TYPE_BEGIN, name)
#define COZ_END(name) COZ_INCREMENT_COUNTER(COZ_COUNTER_TYPE_END, name)
#if defined(__cplusplus)
}
#endif
#endif

191
Programs/hugePage/GetInfo.c Normal file
View File

@@ -0,0 +1,191 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/mman.h>
#include <string.h>
#include<stdio.h>
// #include "common/rte_log.h"
#include <fcntl.h>
// #include "common/eal_private.h"
// #include "common/eal_hugepages.h"
// #include "common/eal_internal_cfg.h"
// #include "common/eal_filesystem.h"
#define CONTIGMEM_DEV "/dev/contigmem"
/* Debug printf */
#define dprintf(...) fprintf(stdout, __VA_ARGS__)
/*
* Uses mmap to create a shared memory area for storage of data
* Used in this file to store the hugepage file map on disk
*/
static void *
map_shared_memory(const char *filename, const size_t mem_size, int flags)
{
void *retval;
int fd = open(filename, flags, 0600);
if (fd < 0)
return NULL;
if (ftruncate(fd, mem_size) < 0) {
close(fd);
return NULL;
}
retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
close(fd);
return retval == MAP_FAILED ? NULL : retval;
}
static void *
open_shared_memory(const char *filename, const size_t mem_size)
{
return map_shared_memory(filename, mem_size, O_RDWR);
}
static void *
create_shared_memory(const char *filename, const size_t mem_size)
{
return map_shared_memory(filename, mem_size, O_RDWR | O_CREAT);
}
/*
* No hugepage support on freebsd, but we dummy it, using contigmem driver
*/
int
eal_hugepage_info_init(void)
{
size_t sysctl_size;
int* num_buffers, fd, error;
int64_t buffer_size;
// struct internal_config *internal_conf =
// eal_get_internal_configuration();
// /* re-use the linux "internal config" structure for our memory data */
// struct hugepage_info *hpi = &internal_conf->hugepage_info[0];
// struct hugepage_info *tmp_hpi;
// unsigned int i;
// internal_conf->num_hugepage_sizes = 1;
sysctl_size = sizeof(num_buffers);
// printf(num_buffers);
error = sysctlbyname("hw.contigmem.num_buffers", &num_buffers,
&sysctl_size, NULL, 0);
// printf(num_buffers);
// printf(sysctl_size);
// int buffers = *num_buffers;
// dprintf(buffers);
if (error != 0) {
// RTE_LOG(ERR, EAL, "could not read sysctl hw.contigmem.num_buffers\n");
return -1;
}
sysctl_size = sizeof(buffer_size);
error = sysctlbyname("hw.contigmem.buffer_size", &buffer_size,
&sysctl_size, NULL, 0);
if (error != 0) {
// RTE_LOG(ERR, EAL, "could not read sysctl hw.contigmem.buffer_size\n");
return -1;
}
// fd = open(CONTIGMEM_DEV, O_RDWR);
// if (fd < 0) {
// RTE_LOG(ERR, EAL, "could not open "CONTIGMEM_DEV"\n");
// return -1;
// }
// if (flock(fd, LOCK_EX | LOCK_NB) < 0) {
// RTE_LOG(ERR, EAL, "could not lock memory. Is another DPDK process running?\n");
// return -1;
// }
// if (buffer_size >= 1<<30)
// RTE_LOG(INFO, EAL, "Contigmem driver has %d buffers, each of size %dGB\n",
// num_buffers, (int)(buffer_size>>30));
// else if (buffer_size >= 1<<20)
// RTE_LOG(INFO, EAL, "Contigmem driver has %d buffers, each of size %dMB\n",
// num_buffers, (int)(buffer_size>>20));
// else
// RTE_LOG(INFO, EAL, "Contigmem driver has %d buffers, each of size %dKB\n",
// num_buffers, (int)(buffer_size>>10));
// strlcpy(hpi->hugedir, CONTIGMEM_DEV, sizeof(hpi->hugedir));
// hpi->hugepage_sz = buffer_size;
// hpi->num_pages[0] = num_buffers;
// hpi->lock_descriptor = fd;
// /* for no shared files mode, do not create shared memory config */
// if (internal_conf->no_shconf)
// return 0;
// tmp_hpi = create_shared_memory(eal_hugepage_info_path(),
// sizeof(internal_conf->hugepage_info));
// if (tmp_hpi == NULL ) {
// RTE_LOG(ERR, EAL, "Failed to create shared memory!\n");
// return -1;
// }
// memcpy(tmp_hpi, hpi, sizeof(internal_conf->hugepage_info));
// /* we've copied file descriptors along with everything else, but they
// * will be invalid in secondary process, so overwrite them
// */
// for (i = 0; i < RTE_DIM(internal_conf->hugepage_info); i++) {
// struct hugepage_info *tmp = &tmp_hpi[i];
// tmp->lock_descriptor = -1;
// }
// if (munmap(tmp_hpi, sizeof(internal_conf->hugepage_info)) < 0) {
// RTE_LOG(ERR, EAL, "Failed to unmap shared memory!\n");
// return -1;
// }
return 0;
}
/* copy stuff from shared info into internal config */
// int
// eal_hugepage_info_read(void)
// {
// struct internal_config *internal_conf =
// eal_get_internal_configuration();
// struct hugepage_info *hpi = &internal_conf->hugepage_info[0];
// struct hugepage_info *tmp_hpi;
// internal_conf->num_hugepage_sizes = 1;
// tmp_hpi = open_shared_memory(eal_hugepage_info_path(),
// sizeof(internal_conf->hugepage_info));
// if (tmp_hpi == NULL) {
// RTE_LOG(ERR, EAL, "Failed to open shared memory!\n");
// return -1;
// }
// memcpy(hpi, tmp_hpi, sizeof(internal_conf->hugepage_info));
// if (munmap(tmp_hpi, sizeof(internal_conf->hugepage_info)) < 0) {
// RTE_LOG(ERR, EAL, "Failed to unmap shared memory!\n");
// return -1;
// }
// return 0;
// }
int main(int argc, char const *argv[])
{
eal_hugepage_info_init();
// eal_hugepage_info_read();
/* code */
return 0;
}

3
Programs/hugePage/build.sh Normal file
View File

@@ -0,0 +1,3 @@
git pull origin main
cc -g -Wall -o test.out -mabi=purecap-benchmark GetInfo.c -v
./test.out

309
Programs/hugePage/common/bus_driver.h Normal file
View File

@@ -0,0 +1,309 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 Red Hat, Inc.
*/
#ifndef BUS_DRIVER_H
#define BUS_DRIVER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "rte_bus.h"
#include "rte_compat.h"
#include "rte_dev.h"
#include "rte_eal.h"
#include "rte_tailq.h>
struct rte_devargs;
struct rte_device;
/** Double linked list of buses */
RTE_TAILQ_HEAD(rte_bus_list, rte_bus);
/**
* Bus specific scan for devices attached on the bus.
* For each bus object, the scan would be responsible for finding devices and
* adding them to its private device list.
*
* A bus should mandatorily implement this method.
*
* @return
* 0 for successful scan
* <0 for unsuccessful scan with error value
*/
typedef int (*rte_bus_scan_t)(void);
/**
* Implementation specific probe function which is responsible for linking
* devices on that bus with applicable drivers.
*
* This is called while iterating over each registered bus.
*
* @return
* 0 for successful probe
* !0 for any error while probing
*/
typedef int (*rte_bus_probe_t)(void);
/**
* Device iterator to find a device on a bus.
*
* This function returns an rte_device if one of those held by the bus
* matches the data passed as parameter.
*
* If the comparison function returns zero this function should stop iterating
* over any more devices. To continue a search the device of a previous search
* can be passed via the start parameter.
*
* @param cmp
* Comparison function.
*
* @param data
* Data to compare each device against.
*
* @param start
* starting point for the iteration
*
* @return
* The first device matching the data, NULL if none exists.
*/
typedef struct rte_device *
(*rte_bus_find_device_t)(const struct rte_device *start, rte_dev_cmp_t cmp,
const void *data);
/**
* Implementation specific probe function which is responsible for linking
* devices on that bus with applicable drivers.
*
* @param dev
* Device pointer that was returned by a previous call to find_device.
*
* @return
* 0 on success.
* !0 on error.
*/
typedef int (*rte_bus_plug_t)(struct rte_device *dev);
/**
* Implementation specific remove function which is responsible for unlinking
* devices on that bus from assigned driver.
*
* @param dev
* Device pointer that was returned by a previous call to find_device.
*
* @return
* 0 on success.
* !0 on error.
*/
typedef int (*rte_bus_unplug_t)(struct rte_device *dev);
/**
* Bus specific parsing function.
* Validates the syntax used in the textual representation of a device,
* If the syntax is valid and ``addr`` is not NULL, writes the bus-specific
* device representation to ``addr``.
*
* @param[in] name
* device textual description
*
* @param[out] addr
* device information location address, into which parsed info
* should be written. If NULL, nothing should be written, which
* is not an error.
*
* @return
* 0 if parsing was successful.
* !0 for any error.
*/
typedef int (*rte_bus_parse_t)(const char *name, void *addr);
/**
* Parse bus part of the device arguments.
*
* The field name of the struct rte_devargs will be set.
*
* @param da
* Pointer to the devargs to parse.
*
* @return
* 0 on successful parsing, otherwise rte_errno is set.
* -EINVAL: on parsing error.
* -ENODEV: if no key matching a device argument is specified.
* -E2BIG: device name is too long.
*/
typedef int (*rte_bus_devargs_parse_t)(struct rte_devargs *da);
/**
* Device level DMA map function.
* After a successful call, the memory segment will be mapped to the
* given device.
*
* @param dev
* Device pointer.
* @param addr
* Virtual address to map.
* @param iova
* IOVA address to map.
* @param len
* Length of the memory segment being mapped.
*
* @return
* 0 if mapping was successful.
* Negative value and rte_errno is set otherwise.
*/
typedef int (*rte_dev_dma_map_t)(struct rte_device *dev, void *addr,
uint64_t iova, size_t len);
/**
* Device level DMA unmap function.
* After a successful call, the memory segment will no longer be
* accessible by the given device.
*
* @param dev
* Device pointer.
* @param addr
* Virtual address to unmap.
* @param iova
* IOVA address to unmap.
* @param len
* Length of the memory segment being mapped.
*
* @return
* 0 if un-mapping was successful.
* Negative value and rte_errno is set otherwise.
*/
typedef int (*rte_dev_dma_unmap_t)(struct rte_device *dev, void *addr,
uint64_t iova, size_t len);
/**
* Implement a specific hot-unplug handler, which is responsible for
* handle the failure when device be hot-unplugged. When the event of
* hot-unplug be detected, it could call this function to handle
* the hot-unplug failure and avoid app crash.
* @param dev
* Pointer of the device structure.
*
* @return
* 0 on success.
* !0 on error.
*/
typedef int (*rte_bus_hot_unplug_handler_t)(struct rte_device *dev);
/**
* Implement a specific sigbus handler, which is responsible for handling
* the sigbus error which is either original memory error, or specific memory
* error that caused of device be hot-unplugged. When sigbus error be captured,
* it could call this function to handle sigbus error.
* @param failure_addr
* Pointer of the fault address of the sigbus error.
*
* @return
* 0 for success handle the sigbus for hot-unplug.
* 1 for not process it, because it is a generic sigbus error.
* -1 for failed to handle the sigbus for hot-unplug.
*/
typedef int (*rte_bus_sigbus_handler_t)(const void *failure_addr);
/**
* Implementation specific cleanup function which is responsible for cleaning up
* devices on that bus with applicable drivers.
*
* This is called while iterating over each registered bus.
*
* @return
* 0 for successful cleanup
* !0 for any error during cleanup
*/
typedef int (*rte_bus_cleanup_t)(void);
/**
* Bus scan policies
*/
enum rte_bus_scan_mode {
RTE_BUS_SCAN_UNDEFINED,
RTE_BUS_SCAN_ALLOWLIST,
RTE_BUS_SCAN_BLOCKLIST,
};
/**
* A structure used to configure bus operations.
*/
struct rte_bus_conf {
enum rte_bus_scan_mode scan_mode; /**< Scan policy. */
};
/**
* Get common iommu class of the all the devices on the bus. The bus may
* check that those devices are attached to iommu driver.
* If no devices are attached to the bus. The bus may return with don't care
* (_DC) value.
* Otherwise, The bus will return appropriate _pa or _va iova mode.
*
* @return
* enum rte_iova_mode value.
*/
typedef enum rte_iova_mode (*rte_bus_get_iommu_class_t)(void);
/**
* A structure describing a generic bus.
*/
struct rte_bus {
RTE_TAILQ_ENTRY(rte_bus) next; /**< Next bus object in linked list */
const char *name; /**< Name of the bus */
rte_bus_scan_t scan; /**< Scan for devices attached to bus */
rte_bus_probe_t probe; /**< Probe devices on bus */
rte_bus_find_device_t find_device; /**< Find a device on the bus */
rte_bus_plug_t plug; /**< Probe single device for drivers */
rte_bus_unplug_t unplug; /**< Remove single device from driver */
rte_bus_parse_t parse; /**< Parse a device name */
rte_bus_devargs_parse_t devargs_parse; /**< Parse bus devargs */
rte_dev_dma_map_t dma_map; /**< DMA map for device in the bus */
rte_dev_dma_unmap_t dma_unmap; /**< DMA unmap for device in the bus */
struct rte_bus_conf conf; /**< Bus configuration */
rte_bus_get_iommu_class_t get_iommu_class; /**< Get iommu class */
rte_dev_iterate_t dev_iterate; /**< Device iterator. */
rte_bus_hot_unplug_handler_t hot_unplug_handler;
/**< handle hot-unplug failure on the bus */
rte_bus_sigbus_handler_t sigbus_handler;
/**< handle sigbus error on the bus */
rte_bus_cleanup_t cleanup; /**< Cleanup devices on bus */
};
/**
* Register a Bus handler.
*
* @param bus
* A pointer to a rte_bus structure describing the bus
* to be registered.
*/
__rte_internal
void rte_bus_register(struct rte_bus *bus);
/**
* Helper for Bus registration.
* The constructor has higher priority than PMD constructors.
*/
#define RTE_REGISTER_BUS(nm, bus) \
RTE_INIT_PRIO(businitfn_ ##nm, BUS) \
{\
(bus).name = RTE_STR(nm);\
rte_bus_register(&bus); \
}
/**
* Unregister a Bus handler.
*
* @param bus
* A pointer to a rte_bus structure describing the bus
* to be unregistered.
*/
__rte_internal
void rte_bus_unregister(struct rte_bus *bus);
#ifdef __cplusplus
}
#endif
#endif /* BUS_DRIVER_H */

41
Programs/hugePage/common/dev_driver.h Normal file
View File

@@ -0,0 +1,41 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 Red Hat, Inc.
*/
#ifndef DEV_DRIVER_H
#define DEV_DRIVER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "rte_common.h"
#include "rte_dev.h"
/**
* A structure describing a device driver.
*/
struct rte_driver {
struct rte_driver *next; /**< Next in list. */
const char *name; /**< Driver name. */
const char *alias; /**< Driver alias. */
};
/**
* A structure describing a generic device.
*/
struct rte_device {
struct rte_device *next; /**< Next device */
const char *name; /**< Device name */
const char *bus_info; /**< Device bus specific information */
const struct rte_driver *driver; /**< Driver assigned after probing */
const struct rte_bus *bus; /**< Bus handle assigned on scan */
int numa_node; /**< NUMA node connection */
struct rte_devargs *devargs; /**< Arguments for latest probing */
};
#ifdef __cplusplus
}
#endif
#endif /* DEV_DRIVER_H */

302
Programs/hugePage/common/eal_common_bus.c Normal file
View File

@@ -0,0 +1,302 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2016 NXP
*/
#include <stdio.h>
#include <string.h>
#include <sys/queue.h>
#include <bus_driver.h>
#include "rte_debug.h"
#include "rte_string_fns.h"
#include "rte_errno.h"
#include "eal_private.h"
static struct rte_bus_list rte_bus_list =
TAILQ_HEAD_INITIALIZER(rte_bus_list);
const char *
rte_bus_name(const struct rte_bus *bus)
{
return bus->name;
}
void
rte_bus_register(struct rte_bus *bus)
{
RTE_VERIFY(bus);
RTE_VERIFY(rte_bus_name(bus) && strlen(rte_bus_name(bus)));
/* A bus should mandatorily have the scan implemented */
RTE_VERIFY(bus->scan);
RTE_VERIFY(bus->probe);
RTE_VERIFY(bus->find_device);
/* Buses supporting driver plug also require unplug. */
RTE_VERIFY(!bus->plug || bus->unplug);
TAILQ_INSERT_TAIL(&rte_bus_list, bus, next);
RTE_LOG(DEBUG, EAL, "Registered [%s] bus.\n", rte_bus_name(bus));
}
void
rte_bus_unregister(struct rte_bus *bus)
{
TAILQ_REMOVE(&rte_bus_list, bus, next);
RTE_LOG(DEBUG, EAL, "Unregistered [%s] bus.\n", rte_bus_name(bus));
}
/* Scan all the buses for registered devices */
int
rte_bus_scan(void)
{
int ret;
struct rte_bus *bus = NULL;
TAILQ_FOREACH(bus, &rte_bus_list, next) {
ret = bus->scan();
if (ret)
RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
rte_bus_name(bus));
}
return 0;
}
/* Probe all devices of all buses */
int
rte_bus_probe(void)
{
int ret;
struct rte_bus *bus, *vbus = NULL;
TAILQ_FOREACH(bus, &rte_bus_list, next) {
if (!strcmp(rte_bus_name(bus), "vdev")) {
vbus = bus;
continue;
}
ret = bus->probe();
if (ret)
RTE_LOG(ERR, EAL, "Bus (%s) probe failed.\n",
rte_bus_name(bus));
}
if (vbus) {
ret = vbus->probe();
if (ret)
RTE_LOG(ERR, EAL, "Bus (%s) probe failed.\n",
rte_bus_name(vbus));
}
return 0;
}
/* Clean up all devices of all buses */
int
eal_bus_cleanup(void)
{
int ret = 0;
struct rte_bus *bus;
TAILQ_FOREACH(bus, &rte_bus_list, next) {
if (bus->cleanup == NULL)
continue;
if (bus->cleanup() != 0)
ret = -1;
}
return ret;
}
/* Dump information of a single bus */
static int
bus_dump_one(FILE *f, struct rte_bus *bus)
{
int ret;
/* For now, dump only the bus name */
ret = fprintf(f, " %s\n", rte_bus_name(bus));
/* Error in case of inability in writing to stream */
if (ret < 0)
return ret;
return 0;
}
void
rte_bus_dump(FILE *f)
{
int ret;
struct rte_bus *bus;
TAILQ_FOREACH(bus, &rte_bus_list, next) {
ret = bus_dump_one(f, bus);
if (ret) {
RTE_LOG(ERR, EAL, "Unable to write to stream (%d)\n",
ret);
break;
}
}
}
struct rte_bus *
rte_bus_find(const struct rte_bus *start, rte_bus_cmp_t cmp,
const void *data)
{
struct rte_bus *bus;
if (start != NULL)
bus = TAILQ_NEXT(start, next);
else
bus = TAILQ_FIRST(&rte_bus_list);
while (bus != NULL) {
if (cmp(bus, data) == 0)
break;
bus = TAILQ_NEXT(bus, next);
}
return bus;
}
static int
cmp_rte_device(const struct rte_device *dev1, const void *_dev2)
{
const struct rte_device *dev2 = _dev2;
return dev1 != dev2;
}
static int
bus_find_device(const struct rte_bus *bus, const void *_dev)
{
struct rte_device *dev;
dev = bus->find_device(NULL, cmp_rte_device, _dev);
return dev == NULL;
}
struct rte_bus *
rte_bus_find_by_device(const struct rte_device *dev)
{
return rte_bus_find(NULL, bus_find_device, (const void *)dev);
}
static int
cmp_bus_name(const struct rte_bus *bus, const void *_name)
{
const char *name = _name;
return strcmp(rte_bus_name(bus), name);
}
struct rte_bus *
rte_bus_find_by_name(const char *busname)
{
return rte_bus_find(NULL, cmp_bus_name, (const void *)busname);
}
static int
bus_can_parse(const struct rte_bus *bus, const void *_name)
{
const char *name = _name;
return !(bus->parse && bus->parse(name, NULL) == 0);
}
struct rte_bus *
rte_bus_find_by_device_name(const char *str)
{
char name[RTE_DEV_NAME_MAX_LEN];
char *c;
strlcpy(name, str, sizeof(name));
c = strchr(name, ',');
if (c != NULL)
c[0] = '\0';
return rte_bus_find(NULL, bus_can_parse, name);
}
/*
* Get iommu class of devices on the bus.
*/
enum rte_iova_mode
rte_bus_get_iommu_class(void)
{
enum rte_iova_mode mode = RTE_IOVA_DC;
bool buses_want_va = false;
bool buses_want_pa = false;
struct rte_bus *bus;
TAILQ_FOREACH(bus, &rte_bus_list, next) {
enum rte_iova_mode bus_iova_mode;
if (bus->get_iommu_class == NULL)
continue;
bus_iova_mode = bus->get_iommu_class();
RTE_LOG(DEBUG, EAL, "Bus %s wants IOVA as '%s'\n",
rte_bus_name(bus),
bus_iova_mode == RTE_IOVA_DC ? "DC" :
(bus_iova_mode == RTE_IOVA_PA ? "PA" : "VA"));
if (bus_iova_mode == RTE_IOVA_PA)
buses_want_pa = true;
else if (bus_iova_mode == RTE_IOVA_VA)
buses_want_va = true;
}
if (buses_want_va && !buses_want_pa) {
mode = RTE_IOVA_VA;
} else if (buses_want_pa && !buses_want_va) {
mode = RTE_IOVA_PA;
} else {
mode = RTE_IOVA_DC;
if (buses_want_va) {
RTE_LOG(WARNING, EAL, "Some buses want 'VA' but forcing 'DC' because other buses want 'PA'.\n");
RTE_LOG(WARNING, EAL, "Depending on the final decision by the EAL, not all buses may be able to initialize.\n");
}
}
return mode;
}
static int
bus_handle_sigbus(const struct rte_bus *bus,
const void *failure_addr)
{
int ret;
if (!bus->sigbus_handler)
return -1;
ret = bus->sigbus_handler(failure_addr);
/* find bus but handle failed, keep the errno be set. */
if (ret < 0 && rte_errno == 0)
rte_errno = ENOTSUP;
return ret > 0;
}
int
rte_bus_sigbus_handler(const void *failure_addr)
{
struct rte_bus *bus;
int ret = 0;
int old_errno = rte_errno;
rte_errno = 0;
bus = rte_bus_find(NULL, bus_handle_sigbus, failure_addr);
/* can not find bus. */
if (!bus)
return 1;
/* find bus but handle failed, pass on the new errno. */
else if (rte_errno != 0)
return -1;
/* restore the old errno. */
rte_errno = old_errno;
return ret;
}

62
Programs/hugePage/common/eal_common_class.c Normal file
View File

@@ -0,0 +1,62 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2018 Gaëtan Rivet
*/
#include <stdio.h>
#include <string.h>
#include <sys/queue.h>
#include "rte_class.h"
#include "rte_debug.h"
static struct rte_class_list rte_class_list =
TAILQ_HEAD_INITIALIZER(rte_class_list);
void
rte_class_register(struct rte_class *class)
{
RTE_VERIFY(class);
RTE_VERIFY(class->name && strlen(class->name));
TAILQ_INSERT_TAIL(&rte_class_list, class, next);
RTE_LOG(DEBUG, EAL, "Registered [%s] device class.\n", class->name);
}
void
rte_class_unregister(struct rte_class *class)
{
TAILQ_REMOVE(&rte_class_list, class, next);
RTE_LOG(DEBUG, EAL, "Unregistered [%s] device class.\n", class->name);
}
struct rte_class *
rte_class_find(const struct rte_class *start, rte_class_cmp_t cmp,
const void *data)
{
struct rte_class *cls;
if (start != NULL)
cls = TAILQ_NEXT(start, next);
else
cls = TAILQ_FIRST(&rte_class_list);
while (cls != NULL) {
if (cmp(cls, data) == 0)
break;
cls = TAILQ_NEXT(cls, next);
}
return cls;
}
static int
cmp_class_name(const struct rte_class *class, const void *_name)
{
const char *name = _name;
return strcmp(class->name, name);
}
struct rte_class *
rte_class_find_by_name(const char *name)
{
return rte_class_find(NULL, cmp_class_name, (const void *)name);
}

94
Programs/hugePage/common/eal_common_config.c Normal file
View File

@@ -0,0 +1,94 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2020 Mellanox Technologies, Ltd
*/
#include "rte_string_fns.h"
#include "eal_private.h"
#include "eal_memcfg.h"
/* early configuration structure, when memory config is not mmapped */
static struct rte_mem_config early_mem_config;
/* Address of global and public configuration */
static struct rte_config rte_config = {
.mem_config = &early_mem_config,
};
/* platform-specific runtime dir */
static char runtime_dir[PATH_MAX];
/* internal configuration */
static struct internal_config internal_config;
const char *
rte_eal_get_runtime_dir(void)
{
return runtime_dir;
}
int
eal_set_runtime_dir(const char *run_dir)
{
if (strlcpy(runtime_dir, run_dir, PATH_MAX) >= PATH_MAX) {
RTE_LOG(ERR, EAL, "Runtime directory string too long\n");
return -1;
}
return 0;
}
/* Return a pointer to the configuration structure */
struct rte_config *
rte_eal_get_configuration(void)
{
return &rte_config;
}
/* Return a pointer to the internal configuration structure */
struct internal_config *
eal_get_internal_configuration(void)
{
return &internal_config;
}
enum rte_iova_mode
rte_eal_iova_mode(void)
{
return rte_eal_get_configuration()->iova_mode;
}
/* Get the EAL base address */
uint64_t
rte_eal_get_baseaddr(void)
{
return (internal_config.base_virtaddr != 0) ?
(uint64_t) internal_config.base_virtaddr :
eal_get_baseaddr();
}
enum rte_proc_type_t
rte_eal_process_type(void)
{
return rte_config.process_type;
}
/* Return user provided mbuf pool ops name */
const char *
rte_eal_mbuf_user_pool_ops(void)
{
return internal_config.user_mbuf_pool_ops_name;
}
/* return non-zero if hugepages are enabled. */
int
rte_eal_has_hugepages(void)
{
return !internal_config.no_hugetlbfs;
}
int
rte_eal_has_pci(void)
{
return !internal_config.no_pci;
}

41
Programs/hugePage/common/eal_common_cpuflags.c Normal file
View File

@@ -0,0 +1,41 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include "rte_common.h"
#include "rte_cpuflags.h"
int
rte_cpu_is_supported(void)
{
/* This is generated at compile-time by the build system */
static const enum rte_cpu_flag_t compile_time_flags[] = {
RTE_COMPILE_TIME_CPUFLAGS
};
unsigned count = RTE_DIM(compile_time_flags), i;
int ret;
for (i = 0; i < count; i++) {
ret = rte_cpu_get_flag_enabled(compile_time_flags[i]);
if (ret < 0) {
fprintf(stderr,
"ERROR: CPU feature flag lookup failed with error %d\n",
ret);
return 0;
}
#ifndef __FreeBSD__
if (!ret) {
fprintf(stderr,
"ERROR: This system does not support \"%s\".\n"
"Please check that RTE_MACHINE is set correctly.\n",
rte_cpu_get_flag_name(compile_time_flags[i]));
return 0;
}
#endif
}
return 1;
}

46
Programs/hugePage/common/eal_common_debug.c Normal file
View File

@@ -0,0 +1,46 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdarg.h>
#include <stdlib.h>
#include "rte_eal.h"
#include "rte_log.h"
#include "rte_debug.h"
void
__rte_panic(const char *funcname, const char *format, ...)
{
va_list ap;
rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_EAL, "PANIC in %s():\n", funcname);
va_start(ap, format);
rte_vlog(RTE_LOG_CRIT, RTE_LOGTYPE_EAL, format, ap);
va_end(ap);
rte_dump_stack();
abort(); /* generate a coredump if enabled */
}
/*
* Like rte_panic this terminates the application. However, no traceback is
* provided and no core-dump is generated.
*/
void
rte_exit(int exit_code, const char *format, ...)
{
va_list ap;
if (exit_code != 0)
RTE_LOG(CRIT, EAL, "Error - exiting with code: %d\n"
" Cause: ", exit_code);
va_start(ap, format);
rte_vlog(RTE_LOG_CRIT, RTE_LOGTYPE_EAL, format, ap);
va_end(ap);
if (rte_eal_cleanup() != 0)
RTE_LOG(CRIT, EAL,
"EAL could not release all resources\n");
exit(exit_code);
}

840
Programs/hugePage/common/eal_common_dev.c Normal file
View File

@@ -0,0 +1,840 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation.
* Copyright(c) 2014 6WIND S.A.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/queue.h>
#include <bus_driver.h>
#include "rte_class.h"
#include <dev_driver.h>
#include "rte_devargs.h"
#include "rte_errno.h"
#include "rte_log.h"
#include "rte_spinlock.h"
#include "rte_string_fns.h"
#include "eal_private.h"
#include "hotplug_mp.h"
const char *
rte_driver_name(const struct rte_driver *driver)
{
return driver->name;
}
const struct rte_bus *
rte_dev_bus(const struct rte_device *dev)
{
return dev->bus;
}
const char *
rte_dev_bus_info(const struct rte_device *dev)
{
return dev->bus_info;
}
const struct rte_devargs *
rte_dev_devargs(const struct rte_device *dev)
{
return dev->devargs;
}
const struct rte_driver *
rte_dev_driver(const struct rte_device *dev)
{
return dev->driver;
}
const char *
rte_dev_name(const struct rte_device *dev)
{
return dev->name;
}
int
rte_dev_numa_node(const struct rte_device *dev)
{
return dev->numa_node;
}
/**
* The device event callback description.
*
* It contains callback address to be registered by user application,
* the pointer to the parameters for callback, and the device name.
*/
struct dev_event_callback {
TAILQ_ENTRY(dev_event_callback) next; /**< Callbacks list */
rte_dev_event_cb_fn cb_fn; /**< Callback address */
void *cb_arg; /**< Callback parameter */
char *dev_name; /**< Callback device name, NULL is for all device */
uint32_t active; /**< Callback is executing */
};
/** @internal Structure to keep track of registered callbacks */
TAILQ_HEAD(dev_event_cb_list, dev_event_callback);
/* The device event callback list for all registered callbacks. */
static struct dev_event_cb_list dev_event_cbs;
/* spinlock for device callbacks */
static rte_spinlock_t dev_event_lock = RTE_SPINLOCK_INITIALIZER;
struct dev_next_ctx {
struct rte_dev_iterator *it;
const char *bus_str;
const char *cls_str;
};
#define CTX(it, bus_str, cls_str) \
(&(const struct dev_next_ctx){ \
.it = it, \
.bus_str = bus_str, \
.cls_str = cls_str, \
})
#define ITCTX(ptr) \
(((struct dev_next_ctx *)(intptr_t)ptr)->it)
#define BUSCTX(ptr) \
(((struct dev_next_ctx *)(intptr_t)ptr)->bus_str)
#define CLSCTX(ptr) \
(((struct dev_next_ctx *)(intptr_t)ptr)->cls_str)
static int cmp_dev_name(const struct rte_device *dev, const void *_name)
{
const char *name = _name;
return strcmp(dev->name, name);
}
int
rte_dev_is_probed(const struct rte_device *dev)
{
/* The field driver should be set only when the probe is successful. */
return dev->driver != NULL;
}
/* helper function to build devargs, caller should free the memory */
static int
build_devargs(const char *busname, const char *devname,
const char *drvargs, char **devargs)
{
int length;
length = snprintf(NULL, 0, "%s:%s,%s", busname, devname, drvargs);
if (length < 0)
return -EINVAL;
*devargs = malloc(length + 1);
if (*devargs == NULL)
return -ENOMEM;
length = snprintf(*devargs, length + 1, "%s:%s,%s",
busname, devname, drvargs);
if (length < 0) {
free(*devargs);
return -EINVAL;
}
return 0;
}
int
rte_eal_hotplug_add(const char *busname, const char *devname,
const char *drvargs)
{
char *devargs;
int ret;
ret = build_devargs(busname, devname, drvargs, &devargs);
if (ret != 0)
return ret;
ret = rte_dev_probe(devargs);
free(devargs);
return ret;
}
/* probe device at local process. */
int
local_dev_probe(const char *devargs, struct rte_device **new_dev)
{
struct rte_device *dev;
struct rte_devargs *da;
int ret;
*new_dev = NULL;
da = calloc(1, sizeof(*da));
if (da == NULL)
return -ENOMEM;
ret = rte_devargs_parse(da, devargs);
if (ret)
goto err_devarg;
if (da->bus->plug == NULL) {
RTE_LOG(ERR, EAL, "Function plug not supported by bus (%s)\n",
da->bus->name);
ret = -ENOTSUP;
goto err_devarg;
}
ret = rte_devargs_insert(&da);
if (ret)
goto err_devarg;
/* the rte_devargs will be referenced in the matching rte_device */
ret = da->bus->scan();
if (ret)
goto err_devarg;
dev = da->bus->find_device(NULL, cmp_dev_name, da->name);
if (dev == NULL) {
RTE_LOG(ERR, EAL, "Cannot find device (%s)\n",
da->name);
ret = -ENODEV;
goto err_devarg;
}
/* Since there is a matching device, it is now its responsibility
* to manage the devargs we've just inserted. From this point
* those devargs shouldn't be removed manually anymore.
*/
ret = dev->bus->plug(dev);
if (ret > 0)
ret = -ENOTSUP;
if (ret && !rte_dev_is_probed(dev)) { /* if hasn't ever succeeded */
RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n",
dev->name);
return ret;
}
*new_dev = dev;
return ret;
err_devarg:
if (rte_devargs_remove(da) != 0) {
rte_devargs_reset(da);
free(da);
}
return ret;
}
int
rte_dev_probe(const char *devargs)
{
struct eal_dev_mp_req req;
struct rte_device *dev;
int ret;
memset(&req, 0, sizeof(req));
req.t = EAL_DEV_REQ_TYPE_ATTACH;
strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
/**
* If in secondary process, just send IPC request to
* primary process.
*/
ret = eal_dev_hotplug_request_to_primary(&req);
if (ret != 0) {
RTE_LOG(ERR, EAL,
"Failed to send hotplug request to primary\n");
return -ENOMSG;
}
if (req.result != 0)
RTE_LOG(ERR, EAL,
"Failed to hotplug add device\n");
return req.result;
}
/* attach a shared device from primary start from here: */
/* primary attach the new device itself. */
ret = local_dev_probe(devargs, &dev);
if (ret != 0) {
RTE_LOG(ERR, EAL,
"Failed to attach device on primary process\n");
/**
* it is possible that secondary process failed to attached a
* device that primary process have during initialization,
* so for -EEXIST case, we still need to sync with secondary
* process.
*/
if (ret != -EEXIST)
return ret;
}
/* primary send attach sync request to secondary. */
ret = eal_dev_hotplug_request_to_secondary(&req);
/* if any communication error, we need to rollback. */
if (ret != 0) {
RTE_LOG(ERR, EAL,
"Failed to send hotplug add request to secondary\n");
ret = -ENOMSG;
goto rollback;
}
/**
* if any secondary failed to attach, we need to consider if rollback
* is necessary.
*/
if (req.result != 0) {
RTE_LOG(ERR, EAL,
"Failed to attach device on secondary process\n");
ret = req.result;
/* for -EEXIST, we don't need to rollback. */
if (ret == -EEXIST)
return ret;
goto rollback;
}
return 0;
rollback:
req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK;
/* primary send rollback request to secondary. */
if (eal_dev_hotplug_request_to_secondary(&req) != 0)
RTE_LOG(WARNING, EAL,
"Failed to rollback device attach on secondary."
"Devices in secondary may not sync with primary\n");
/* primary rollback itself. */
if (local_dev_remove(dev) != 0)
RTE_LOG(WARNING, EAL,
"Failed to rollback device attach on primary."
"Devices in secondary may not sync with primary\n");
return ret;
}
int
rte_eal_hotplug_remove(const char *busname, const char *devname)
{
struct rte_device *dev;
struct rte_bus *bus;
bus = rte_bus_find_by_name(busname);
if (bus == NULL) {
RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", busname);
return -ENOENT;
}
dev = bus->find_device(NULL, cmp_dev_name, devname);
if (dev == NULL) {
RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", devname);
return -EINVAL;
}
return rte_dev_remove(dev);
}
/* remove device at local process. */
int
local_dev_remove(struct rte_device *dev)
{
int ret;
if (dev->bus->unplug == NULL) {
RTE_LOG(ERR, EAL, "Function unplug not supported by bus (%s)\n",
dev->bus->name);
return -ENOTSUP;
}
ret = dev->bus->unplug(dev);
if (ret) {
RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n",
dev->name);
return (ret < 0) ? ret : -ENOENT;
}
return 0;
}
int
rte_dev_remove(struct rte_device *dev)
{
struct eal_dev_mp_req req;
char *devargs;
int ret;
if (!rte_dev_is_probed(dev)) {
RTE_LOG(ERR, EAL, "Device is not probed\n");
return -ENOENT;
}
ret = build_devargs(dev->bus->name, dev->name, "", &devargs);
if (ret != 0)
return ret;
memset(&req, 0, sizeof(req));
req.t = EAL_DEV_REQ_TYPE_DETACH;
strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
free(devargs);
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
/**
* If in secondary process, just send IPC request to
* primary process.
*/
ret = eal_dev_hotplug_request_to_primary(&req);
if (ret != 0) {
RTE_LOG(ERR, EAL,
"Failed to send hotplug request to primary\n");
return -ENOMSG;
}
if (req.result != 0)
RTE_LOG(ERR, EAL,
"Failed to hotplug remove device\n");
return req.result;
}
/* detach a device from primary start from here: */
/* primary send detach sync request to secondary */
ret = eal_dev_hotplug_request_to_secondary(&req);
/**
* if communication error, we need to rollback, because it is possible
* part of the secondary processes still detached it successfully.
*/
if (ret != 0) {
RTE_LOG(ERR, EAL,
"Failed to send device detach request to secondary\n");
ret = -ENOMSG;
goto rollback;
}
/**
* if any secondary failed to detach, we need to consider if rollback
* is necessary.
*/
if (req.result != 0) {
RTE_LOG(ERR, EAL,
"Failed to detach device on secondary process\n");
ret = req.result;
/**
* if -ENOENT, we don't need to rollback, since devices is
* already detached on secondary process.
*/
if (ret != -ENOENT)
goto rollback;
}
/* primary detach the device itself. */
ret = local_dev_remove(dev);
/* if primary failed, still need to consider if rollback is necessary */
if (ret != 0) {
RTE_LOG(ERR, EAL,
"Failed to detach device on primary process\n");
/* if -ENOENT, we don't need to rollback */
if (ret == -ENOENT)
return ret;
goto rollback;
}
return 0;
rollback:
req.t = EAL_DEV_REQ_TYPE_DETACH_ROLLBACK;
/* primary send rollback request to secondary. */
if (eal_dev_hotplug_request_to_secondary(&req) != 0)
RTE_LOG(WARNING, EAL,
"Failed to rollback device detach on secondary."
"Devices in secondary may not sync with primary\n");
return ret;
}
int
rte_dev_event_callback_register(const char *device_name,
rte_dev_event_cb_fn cb_fn,
void *cb_arg)
{
struct dev_event_callback *event_cb;
int ret;
if (!cb_fn)
return -EINVAL;
rte_spinlock_lock(&dev_event_lock);
if (TAILQ_EMPTY(&dev_event_cbs))
TAILQ_INIT(&dev_event_cbs);
TAILQ_FOREACH(event_cb, &dev_event_cbs, next) {
if (event_cb->cb_fn == cb_fn && event_cb->cb_arg == cb_arg) {
if (device_name == NULL && event_cb->dev_name == NULL)
break;
if (device_name == NULL || event_cb->dev_name == NULL)
continue;
if (!strcmp(event_cb->dev_name, device_name))
break;
}
}
/* create a new callback. */
if (event_cb == NULL) {
event_cb = malloc(sizeof(struct dev_event_callback));
if (event_cb != NULL) {
event_cb->cb_fn = cb_fn;
event_cb->cb_arg = cb_arg;
event_cb->active = 0;
if (!device_name) {
event_cb->dev_name = NULL;
} else {
event_cb->dev_name = strdup(device_name);
if (event_cb->dev_name == NULL) {
ret = -ENOMEM;
goto error;
}
}
TAILQ_INSERT_TAIL(&dev_event_cbs, event_cb, next);
} else {
RTE_LOG(ERR, EAL,
"Failed to allocate memory for device "
"event callback.");
ret = -ENOMEM;
goto error;
}
} else {
RTE_LOG(ERR, EAL,
"The callback is already exist, no need "
"to register again.\n");
event_cb = NULL;
ret = -EEXIST;
goto error;
}
rte_spinlock_unlock(&dev_event_lock);
return 0;
error:
free(event_cb);
rte_spinlock_unlock(&dev_event_lock);
return ret;
}
int
rte_dev_event_callback_unregister(const char *device_name,
rte_dev_event_cb_fn cb_fn,
void *cb_arg)
{
int ret = 0;
struct dev_event_callback *event_cb, *next;
if (!cb_fn)
return -EINVAL;
rte_spinlock_lock(&dev_event_lock);
/*walk through the callbacks and remove all that match. */
for (event_cb = TAILQ_FIRST(&dev_event_cbs); event_cb != NULL;
event_cb = next) {
next = TAILQ_NEXT(event_cb, next);
if (device_name != NULL && event_cb->dev_name != NULL) {
if (!strcmp(event_cb->dev_name, device_name)) {
if (event_cb->cb_fn != cb_fn ||
(cb_arg != (void *)-1 &&
event_cb->cb_arg != cb_arg))
continue;
}
} else if (device_name != NULL) {
continue;
}
/*
* if this callback is not executing right now,
* then remove it.
*/
if (event_cb->active == 0) {
TAILQ_REMOVE(&dev_event_cbs, event_cb, next);
free(event_cb->dev_name);
free(event_cb);
ret++;
} else {
ret = -EAGAIN;
break;
}
}
/* this callback is not be registered */
if (ret == 0)
ret = -ENOENT;
rte_spinlock_unlock(&dev_event_lock);
return ret;
}
void
rte_dev_event_callback_process(const char *device_name,
enum rte_dev_event_type event)
{
struct dev_event_callback *cb_lst;
if (device_name == NULL)
return;
rte_spinlock_lock(&dev_event_lock);
TAILQ_FOREACH(cb_lst, &dev_event_cbs, next) {
if (cb_lst->dev_name) {
if (strcmp(cb_lst->dev_name, device_name))
continue;
}
cb_lst->active = 1;
rte_spinlock_unlock(&dev_event_lock);
cb_lst->cb_fn(device_name, event,
cb_lst->cb_arg);
rte_spinlock_lock(&dev_event_lock);
cb_lst->active = 0;
}
rte_spinlock_unlock(&dev_event_lock);
}
int
rte_dev_iterator_init(struct rte_dev_iterator *it,
const char *dev_str)
{
struct rte_devargs devargs = { .bus = NULL };
struct rte_class *cls = NULL;
struct rte_bus *bus = NULL;
/* Having both bus_str and cls_str NULL is illegal,
* marking this iterator as invalid unless
* everything goes well.
*/
it->bus_str = NULL;
it->cls_str = NULL;
/* Setting data field implies no malloc in parsing. */
devargs.data = (void *)(intptr_t)dev_str;
if (rte_devargs_layers_parse(&devargs, dev_str))
goto get_out;
bus = devargs.bus;
cls = devargs.cls;
/* The string should have at least
* one layer specified.
*/
if (bus == NULL && cls == NULL) {
RTE_LOG(DEBUG, EAL, "Either bus or class must be specified.\n");
rte_errno = EINVAL;
goto get_out;
}
if (bus != NULL && bus->dev_iterate == NULL) {
RTE_LOG(DEBUG, EAL, "Bus %s not supported\n", bus->name);
rte_errno = ENOTSUP;
goto get_out;
}
if (cls != NULL && cls->dev_iterate == NULL) {
RTE_LOG(DEBUG, EAL, "Class %s not supported\n", cls->name);
rte_errno = ENOTSUP;
goto get_out;
}
it->bus_str = devargs.bus_str;
it->cls_str = devargs.cls_str;
it->dev_str = dev_str;
it->bus = bus;
it->cls = cls;
it->device = NULL;
it->class_device = NULL;
get_out:
return -rte_errno;
}
static char *
dev_str_sane_copy(const char *str)
{
size_t end;
char *copy;
end = strcspn(str, ",/");
if (str[end] == ',') {
copy = strdup(&str[end + 1]);
} else {
/* '/' or '\0' */
copy = strdup("");
}
if (copy == NULL) {
rte_errno = ENOMEM;
} else {
char *slash;
slash = strchr(copy, '/');
if (slash != NULL)
slash[0] = '\0';
}
return copy;
}
static int
class_next_dev_cmp(const struct rte_class *cls,
const void *ctx)
{
struct rte_dev_iterator *it;
const char *cls_str = NULL;
void *dev;
if (cls->dev_iterate == NULL)
return 1;
it = ITCTX(ctx);
cls_str = CLSCTX(ctx);
dev = it->class_device;
/* it->cls_str != NULL means a class
* was specified in the devstr.
*/
if (it->cls_str != NULL && cls != it->cls)
return 1;
/* If an error occurred previously,
* no need to test further.
*/
if (rte_errno != 0)
return -1;
dev = cls->dev_iterate(dev, cls_str, it);
it->class_device = dev;
return dev == NULL;
}
static int
bus_next_dev_cmp(const struct rte_bus *bus,
const void *ctx)
{
struct rte_device *dev = NULL;
struct rte_class *cls = NULL;
struct rte_dev_iterator *it;
const char *bus_str = NULL;
if (bus->dev_iterate == NULL)
return 1;
it = ITCTX(ctx);
bus_str = BUSCTX(ctx);
dev = it->device;
/* it->bus_str != NULL means a bus
* was specified in the devstr.
*/
if (it->bus_str != NULL && bus != it->bus)
return 1;
/* If an error occurred previously,
* no need to test further.
*/
if (rte_errno != 0)
return -1;
if (it->cls_str == NULL) {
dev = bus->dev_iterate(dev, bus_str, it);
goto end;
}
/* cls_str != NULL */
if (dev == NULL) {
next_dev_on_bus:
dev = bus->dev_iterate(dev, bus_str, it);
it->device = dev;
}
if (dev == NULL)
return 1;
if (it->cls != NULL)
cls = TAILQ_PREV(it->cls, rte_class_list, next);
cls = rte_class_find(cls, class_next_dev_cmp, ctx);
if (cls != NULL) {
it->cls = cls;
goto end;
}
goto next_dev_on_bus;
end:
it->device = dev;
return dev == NULL;
}
struct rte_device *
rte_dev_iterator_next(struct rte_dev_iterator *it)
{
struct rte_bus *bus = NULL;
int old_errno = rte_errno;
char *bus_str = NULL;
char *cls_str = NULL;
rte_errno = 0;
if (it->bus_str == NULL && it->cls_str == NULL) {
/* Invalid iterator. */
rte_errno = EINVAL;
return NULL;
}
if (it->bus != NULL)
bus = TAILQ_PREV(it->bus, rte_bus_list, next);
if (it->bus_str != NULL) {
bus_str = dev_str_sane_copy(it->bus_str);
if (bus_str == NULL)
goto out;
}
if (it->cls_str != NULL) {
cls_str = dev_str_sane_copy(it->cls_str);
if (cls_str == NULL)
goto out;
}
while ((bus = rte_bus_find(bus, bus_next_dev_cmp,
CTX(it, bus_str, cls_str)))) {
if (it->device != NULL) {
it->bus = bus;
goto out;
}
if (it->bus_str != NULL ||
rte_errno != 0)
break;
}
if (rte_errno == 0)
rte_errno = old_errno;
out:
free(bus_str);
free(cls_str);
return it->device;
}
int
rte_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
size_t len)
{
if (dev->bus->dma_map == NULL || len == 0) {
rte_errno = ENOTSUP;
return -1;
}
/* Memory must be registered through rte_extmem_* APIs */
if (rte_mem_virt2memseg_list(addr) == NULL) {
rte_errno = EINVAL;
return -1;
}
return dev->bus->dma_map(dev, addr, iova, len);
}
int
rte_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
size_t len)
{
if (dev->bus->dma_unmap == NULL || len == 0) {
rte_errno = ENOTSUP;
return -1;
}
/* Memory must be registered through rte_extmem_* APIs */
if (rte_mem_virt2memseg_list(addr) == NULL) {
rte_errno = EINVAL;
return -1;
}
return dev->bus->dma_unmap(dev, addr, iova, len);
}

426
Programs/hugePage/common/eal_common_devargs.c Normal file
View File

@@ -0,0 +1,426 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2014 6WIND S.A.
*/
/* This file manages the list of devices and their arguments, as given
* by the user at startup
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <bus_driver.h>
#include "rte_class.h"
#include "rte_dev.h"
#include "rte_devargs.h"
#include "rte_errno.h"
#include "rte_kvargs.h"
#include "rte_log.h"
#include "rte_tailq.h"
#include "rte_string_fns.h"
#include "eal_private.h"
/** user device double-linked queue type definition */
TAILQ_HEAD(rte_devargs_list, rte_devargs);
/** Global list of user devices */
static struct rte_devargs_list devargs_list =
TAILQ_HEAD_INITIALIZER(devargs_list);
/* Resolve devargs name from bus arguments. */
static int
devargs_bus_parse_default(struct rte_devargs *devargs,
struct rte_kvargs *bus_args)
{
const char *name;
/* Parse devargs name from bus key-value list. */
name = rte_kvargs_get(bus_args, "name");
if (name == NULL) {
RTE_LOG(DEBUG, EAL, "devargs name not found: %s\n",
devargs->data);
return 0;
}
if (rte_strscpy(devargs->name, name, sizeof(devargs->name)) < 0) {
RTE_LOG(ERR, EAL, "devargs name too long: %s\n",
devargs->data);
return -E2BIG;
}
return 0;
}
int
rte_devargs_layers_parse(struct rte_devargs *devargs,
const char *devstr)
{
struct {
const char *key;
const char *str;
struct rte_kvargs *kvlist;
} layers[] = {
{ RTE_DEVARGS_KEY_BUS "=", NULL, NULL, },
{ RTE_DEVARGS_KEY_CLASS "=", NULL, NULL, },
{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
};
struct rte_kvargs_pair *kv = NULL;
struct rte_kvargs *bus_kvlist = NULL;
char *s;
size_t nblayer = 0;
size_t i;
int ret = 0;
bool allocated_data = false;
/* If the devargs points the devstr
* as source data, then it should not allocate
* anything and keep referring only to it.
*/
if (devargs->data != devstr) {
devargs->data = strdup(devstr);
if (devargs->data == NULL) {
RTE_LOG(ERR, EAL, "OOM\n");
ret = -ENOMEM;
goto get_out;
}
allocated_data = true;
}
s = devargs->data;
while (s != NULL) {
if (nblayer > RTE_DIM(layers)) {
ret = -E2BIG;
goto get_out;
}
layers[nblayer].str = s;
/* Locate next layer starts with valid layer key. */
while (s != NULL) {
s = strchr(s, '/');
if (s == NULL)
break;
for (i = 0; i < RTE_DIM(layers); i++) {
if (strncmp(s + 1, layers[i].key,
strlen(layers[i].key)) == 0) {
*s = '\0';
break;
}
}
s++;
if (i < RTE_DIM(layers))
break;
}
layers[nblayer].kvlist = rte_kvargs_parse
(layers[nblayer].str, NULL);
if (layers[nblayer].kvlist == NULL) {
ret = -EINVAL;
goto get_out;
}
nblayer++;
}
/* Parse each sub-list. */
for (i = 0; i < RTE_DIM(layers); i++) {
if (layers[i].kvlist == NULL)
continue;
kv = &layers[i].kvlist->pairs[0];
if (kv->key == NULL)
continue;
if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
bus_kvlist = layers[i].kvlist;
devargs->bus_str = layers[i].str;
devargs->bus = rte_bus_find_by_name(kv->value);
if (devargs->bus == NULL) {
RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
kv->value);
ret = -EFAULT;
goto get_out;
}
} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
devargs->cls_str = layers[i].str;
devargs->cls = rte_class_find_by_name(kv->value);
if (devargs->cls == NULL) {
RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
kv->value);
ret = -EFAULT;
goto get_out;
}
} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
devargs->drv_str = layers[i].str;
continue;
}
}
/* Resolve devargs name. */
if (devargs->bus != NULL && devargs->bus->devargs_parse != NULL)
ret = devargs->bus->devargs_parse(devargs);
else if (bus_kvlist != NULL)
ret = devargs_bus_parse_default(devargs, bus_kvlist);
get_out:
for (i = 0; i < RTE_DIM(layers); i++) {
rte_kvargs_free(layers[i].kvlist);
}
if (ret != 0) {
if (allocated_data) {
/* Free duplicated data. */
free(devargs->data);
devargs->data = NULL;
}
rte_errno = -ret;
}
return ret;
}
static int
bus_name_cmp(const struct rte_bus *bus, const void *name)
{
return strncmp(bus->name, name, strlen(bus->name));
}
int
rte_devargs_parse(struct rte_devargs *da, const char *dev)
{
struct rte_bus *bus = NULL;
const char *devname;
const size_t maxlen = sizeof(da->name);
size_t i;
if (da == NULL)
return -EINVAL;
memset(da, 0, sizeof(*da));
/* First parse according global device syntax. */
if (rte_devargs_layers_parse(da, dev) == 0) {
if (da->bus != NULL || da->cls != NULL)
return 0;
rte_devargs_reset(da);
}
/* Otherwise fallback to legacy syntax: */
/* Retrieve eventual bus info */
do {
devname = dev;
bus = rte_bus_find(bus, bus_name_cmp, dev);
if (bus == NULL)
break;
devname = dev + strlen(bus->name) + 1;
if (rte_bus_find_by_device_name(devname) == bus)
break;
} while (1);
/* Store device name */
i = 0;
while (devname[i] != '\0' && devname[i] != ',') {
da->name[i] = devname[i];
i++;
if (i == maxlen) {
RTE_LOG(WARNING, EAL, "Parsing \"%s\": device name should be shorter than %zu\n",
dev, maxlen);
da->name[i - 1] = '\0';
return -EINVAL;
}
}
da->name[i] = '\0';
if (bus == NULL) {
bus = rte_bus_find_by_device_name(da->name);
if (bus == NULL) {
RTE_LOG(ERR, EAL, "failed to parse device \"%s\"\n",
da->name);
return -EFAULT;
}
}
da->bus = bus;
/* Parse eventual device arguments */
if (devname[i] == ',')
da->data = strdup(&devname[i + 1]);
else
da->data = strdup("");
if (da->data == NULL) {
RTE_LOG(ERR, EAL, "not enough memory to parse arguments\n");
return -ENOMEM;
}
da->drv_str = da->data;
return 0;
}
int
rte_devargs_parsef(struct rte_devargs *da, const char *format, ...)
{
va_list ap;
int len;
char *dev;
int ret;
if (da == NULL)
return -EINVAL;
va_start(ap, format);
len = vsnprintf(NULL, 0, format, ap);
va_end(ap);
if (len < 0)
return -EINVAL;
len += 1;
dev = calloc(1, (size_t)len);
if (dev == NULL) {
RTE_LOG(ERR, EAL, "not enough memory to parse device\n");
return -ENOMEM;
}
va_start(ap, format);
vsnprintf(dev, (size_t)len, format, ap);
va_end(ap);
ret = rte_devargs_parse(da, dev);
free(dev);
return ret;
}
void
rte_devargs_reset(struct rte_devargs *da)
{
if (da == NULL)
return;
free(da->data);
da->data = NULL;
}
int
rte_devargs_insert(struct rte_devargs **da)
{
struct rte_devargs *listed_da;
void *tmp;
if (*da == NULL || (*da)->bus == NULL)
return -1;
RTE_TAILQ_FOREACH_SAFE(listed_da, &devargs_list, next, tmp) {
if (listed_da == *da)
/* devargs already in the list */
return 0;
if (strcmp(listed_da->bus->name, (*da)->bus->name) == 0 &&
strcmp(listed_da->name, (*da)->name) == 0) {
/* device already in devargs list, must be updated */
(*da)->next = listed_da->next;
rte_devargs_reset(listed_da);
*listed_da = **da;
/* replace provided devargs with found one */
free(*da);
*da = listed_da;
return 0;
}
}
/* new device in the list */
TAILQ_INSERT_TAIL(&devargs_list, *da, next);
return 0;
}
/* store in allowed list parameter for later parsing */
int
rte_devargs_add(enum rte_devtype devtype, const char *devargs_str)
{
struct rte_devargs *devargs = NULL;
struct rte_bus *bus = NULL;
const char *dev = devargs_str;
/* use calloc instead of rte_zmalloc as it's called early at init */
devargs = calloc(1, sizeof(*devargs));
if (devargs == NULL)
goto fail;
if (rte_devargs_parse(devargs, dev))
goto fail;
devargs->type = devtype;
bus = devargs->bus;
if (devargs->type == RTE_DEVTYPE_BLOCKED)
devargs->policy = RTE_DEV_BLOCKED;
if (bus->conf.scan_mode == RTE_BUS_SCAN_UNDEFINED) {
if (devargs->policy == RTE_DEV_ALLOWED)
bus->conf.scan_mode = RTE_BUS_SCAN_ALLOWLIST;
else if (devargs->policy == RTE_DEV_BLOCKED)
bus->conf.scan_mode = RTE_BUS_SCAN_BLOCKLIST;
}
TAILQ_INSERT_TAIL(&devargs_list, devargs, next);
return 0;
fail:
if (devargs) {
rte_devargs_reset(devargs);
free(devargs);
}
return -1;
}
int
rte_devargs_remove(struct rte_devargs *devargs)
{
struct rte_devargs *d;
void *tmp;
if (devargs == NULL || devargs->bus == NULL)
return -1;
RTE_TAILQ_FOREACH_SAFE(d, &devargs_list, next, tmp) {
if (strcmp(d->bus->name, devargs->bus->name) == 0 &&
strcmp(d->name, devargs->name) == 0) {
TAILQ_REMOVE(&devargs_list, d, next);
rte_devargs_reset(d);
free(d);
return 0;
}
}
return 1;
}
/* count the number of devices of a specified type */
unsigned int
rte_devargs_type_count(enum rte_devtype devtype)
{
struct rte_devargs *devargs;
unsigned int count = 0;
TAILQ_FOREACH(devargs, &devargs_list, next) {
if (devargs->type != devtype)
continue;
count++;
}
return count;
}
/* dump the user devices on the console */
void
rte_devargs_dump(FILE *f)
{
struct rte_devargs *devargs;
fprintf(f, "User device list:\n");
TAILQ_FOREACH(devargs, &devargs_list, next) {
fprintf(f, " [%s]: %s %s\n",
(devargs->bus ? devargs->bus->name : "??"),
devargs->name, devargs->args);
}
}
/* bus-aware rte_devargs iterator. */
struct rte_devargs *
rte_devargs_next(const char *busname, const struct rte_devargs *start)
{
struct rte_devargs *da;
if (start != NULL)
da = TAILQ_NEXT(start, next);
else
da = TAILQ_FIRST(&devargs_list);
while (da != NULL) {
if (busname == NULL ||
(strcmp(busname, da->bus->name) == 0))
return da;
da = TAILQ_NEXT(da, next);
}
return NULL;
}

534
Programs/hugePage/common/eal_common_dynmem.c Normal file
View File

@@ -0,0 +1,534 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation.
* Copyright(c) 2013 6WIND S.A.
*/
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include "rte_log.h"
#include "rte_string_fns.h"
#include "eal_internal_cfg.h"
#include "eal_memalloc.h"
#include "eal_memcfg.h"
#include "eal_private.h"
/** @file Functions common to EALs that support dynamic memory allocation. */
int
eal_dynmem_memseg_lists_init(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
struct memtype {
uint64_t page_sz;
int socket_id;
} *memtypes = NULL;
int i, hpi_idx, msl_idx, ret = -1; /* fail unless told to succeed */
struct rte_memseg_list *msl;
uint64_t max_mem, max_mem_per_type;
unsigned int max_seglists_per_type;
unsigned int n_memtypes, cur_type;
struct internal_config *internal_conf =
eal_get_internal_configuration();
/* no-huge does not need this at all */
if (internal_conf->no_hugetlbfs)
return 0;
/*
* figuring out amount of memory we're going to have is a long and very
* involved process. the basic element we're operating with is a memory
* type, defined as a combination of NUMA node ID and page size (so that
* e.g. 2 sockets with 2 page sizes yield 4 memory types in total).
*
* deciding amount of memory going towards each memory type is a
* balancing act between maximum segments per type, maximum memory per
* type, and number of detected NUMA nodes. the goal is to make sure
* each memory type gets at least one memseg list.
*
* the total amount of memory is limited by RTE_MAX_MEM_MB value.
*
* the total amount of memory per type is limited by either
* RTE_MAX_MEM_MB_PER_TYPE, or by RTE_MAX_MEM_MB divided by the number
* of detected NUMA nodes. additionally, maximum number of segments per
* type is also limited by RTE_MAX_MEMSEG_PER_TYPE. this is because for
* smaller page sizes, it can take hundreds of thousands of segments to
* reach the above specified per-type memory limits.
*
* additionally, each type may have multiple memseg lists associated
* with it, each limited by either RTE_MAX_MEM_MB_PER_LIST for bigger
* page sizes, or RTE_MAX_MEMSEG_PER_LIST segments for smaller ones.
*
* the number of memseg lists per type is decided based on the above
* limits, and also taking number of detected NUMA nodes, to make sure
* that we don't run out of memseg lists before we populate all NUMA
* nodes with memory.
*
* we do this in three stages. first, we collect the number of types.
* then, we figure out memory constraints and populate the list of
* would-be memseg lists. then, we go ahead and allocate the memseg
* lists.
*/
/* create space for mem types */
n_memtypes = internal_conf->num_hugepage_sizes * rte_socket_count();
memtypes = calloc(n_memtypes, sizeof(*memtypes));
if (memtypes == NULL) {
RTE_LOG(ERR, EAL, "Cannot allocate space for memory types\n");
return -1;
}
/* populate mem types */
cur_type = 0;
for (hpi_idx = 0; hpi_idx < (int) internal_conf->num_hugepage_sizes;
hpi_idx++) {
struct hugepage_info *hpi;
uint64_t hugepage_sz;
hpi = &internal_conf->hugepage_info[hpi_idx];
hugepage_sz = hpi->hugepage_sz;
for (i = 0; i < (int) rte_socket_count(); i++, cur_type++) {
int socket_id = rte_socket_id_by_idx(i);
#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
/* we can still sort pages by socket in legacy mode */
if (!internal_conf->legacy_mem && socket_id > 0)
break;
#endif
memtypes[cur_type].page_sz = hugepage_sz;
memtypes[cur_type].socket_id = socket_id;
RTE_LOG(DEBUG, EAL, "Detected memory type: "
"socket_id:%u hugepage_sz:%" PRIu64 "\n",
socket_id, hugepage_sz);
}
}
/* number of memtypes could have been lower due to no NUMA support */
n_memtypes = cur_type;
/* set up limits for types */
max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
max_mem_per_type = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20,
max_mem / n_memtypes);
/*
* limit maximum number of segment lists per type to ensure there's
* space for memseg lists for all NUMA nodes with all page sizes
*/
max_seglists_per_type = RTE_MAX_MEMSEG_LISTS / n_memtypes;
if (max_seglists_per_type == 0) {
RTE_LOG(ERR, EAL, "Cannot accommodate all memory types, please increase %s\n",
RTE_STR(RTE_MAX_MEMSEG_LISTS));
goto out;
}
/* go through all mem types and create segment lists */
msl_idx = 0;
for (cur_type = 0; cur_type < n_memtypes; cur_type++) {
unsigned int cur_seglist, n_seglists, n_segs;
unsigned int max_segs_per_type, max_segs_per_list;
struct memtype *type = &memtypes[cur_type];
uint64_t max_mem_per_list, pagesz;
int socket_id;
pagesz = type->page_sz;
socket_id = type->socket_id;
/*
* we need to create segment lists for this type. we must take
* into account the following things:
*
* 1. total amount of memory we can use for this memory type
* 2. total amount of memory per memseg list allowed
* 3. number of segments needed to fit the amount of memory
* 4. number of segments allowed per type
* 5. number of segments allowed per memseg list
* 6. number of memseg lists we are allowed to take up
*/
/* calculate how much segments we will need in total */
max_segs_per_type = max_mem_per_type / pagesz;
/* limit number of segments to maximum allowed per type */
max_segs_per_type = RTE_MIN(max_segs_per_type,
(unsigned int)RTE_MAX_MEMSEG_PER_TYPE);
/* limit number of segments to maximum allowed per list */
max_segs_per_list = RTE_MIN(max_segs_per_type,
(unsigned int)RTE_MAX_MEMSEG_PER_LIST);
/* calculate how much memory we can have per segment list */
max_mem_per_list = RTE_MIN(max_segs_per_list * pagesz,
(uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20);
/* calculate how many segments each segment list will have */
n_segs = RTE_MIN(max_segs_per_list, max_mem_per_list / pagesz);
/* calculate how many segment lists we can have */
n_seglists = RTE_MIN(max_segs_per_type / n_segs,
max_mem_per_type / max_mem_per_list);
/* limit number of segment lists according to our maximum */
n_seglists = RTE_MIN(n_seglists, max_seglists_per_type);
RTE_LOG(DEBUG, EAL, "Creating %i segment lists: "
"n_segs:%i socket_id:%i hugepage_sz:%" PRIu64 "\n",
n_seglists, n_segs, socket_id, pagesz);
/* create all segment lists */
for (cur_seglist = 0; cur_seglist < n_seglists; cur_seglist++) {
if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
RTE_LOG(ERR, EAL,
"No more space in memseg lists, please increase %s\n",
RTE_STR(RTE_MAX_MEMSEG_LISTS));
goto out;
}
msl = &mcfg->memsegs[msl_idx++];
if (eal_memseg_list_init(msl, pagesz, n_segs,
socket_id, cur_seglist, true))
goto out;
if (eal_memseg_list_alloc(msl, 0)) {
RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n");
goto out;
}
}
}
/* we're successful */
ret = 0;
out:
free(memtypes);
return ret;
}
static int __rte_unused
hugepage_count_walk(const struct rte_memseg_list *msl, void *arg)
{
struct hugepage_info *hpi = arg;
if (msl->page_sz != hpi->hugepage_sz)
return 0;
hpi->num_pages[msl->socket_id] += msl->memseg_arr.len;
return 0;
}
static int
limits_callback(int socket_id, size_t cur_limit, size_t new_len)
{
RTE_SET_USED(socket_id);
RTE_SET_USED(cur_limit);
RTE_SET_USED(new_len);
return -1;
}
int
eal_dynmem_hugepage_init(void)
{
struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
uint64_t memory[RTE_MAX_NUMA_NODES];
int hp_sz_idx, socket_id;
struct internal_config *internal_conf =
eal_get_internal_configuration();
memset(used_hp, 0, sizeof(used_hp));
for (hp_sz_idx = 0;
hp_sz_idx < (int) internal_conf->num_hugepage_sizes;
hp_sz_idx++) {
#ifndef RTE_ARCH_64
struct hugepage_info dummy;
unsigned int i;
#endif
/* also initialize used_hp hugepage sizes in used_hp */
struct hugepage_info *hpi;
hpi = &internal_conf->hugepage_info[hp_sz_idx];
used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz;
#ifndef RTE_ARCH_64
/* for 32-bit, limit number of pages on socket to whatever we've
* preallocated, as we cannot allocate more.
*/
memset(&dummy, 0, sizeof(dummy));
dummy.hugepage_sz = hpi->hugepage_sz;
if (rte_memseg_list_walk(hugepage_count_walk, &dummy) < 0)
return -1;
for (i = 0; i < RTE_DIM(dummy.num_pages); i++) {
hpi->num_pages[i] = RTE_MIN(hpi->num_pages[i],
dummy.num_pages[i]);
}
#endif
}
/* make a copy of socket_mem, needed for balanced allocation. */
for (hp_sz_idx = 0; hp_sz_idx < RTE_MAX_NUMA_NODES; hp_sz_idx++)
memory[hp_sz_idx] = internal_conf->socket_mem[hp_sz_idx];
/* calculate final number of pages */
if (eal_dynmem_calc_num_pages_per_socket(memory,
internal_conf->hugepage_info, used_hp,
internal_conf->num_hugepage_sizes) < 0)
return -1;
for (hp_sz_idx = 0;
hp_sz_idx < (int)internal_conf->num_hugepage_sizes;
hp_sz_idx++) {
for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES;
socket_id++) {
struct rte_memseg **pages;
struct hugepage_info *hpi = &used_hp[hp_sz_idx];
unsigned int num_pages = hpi->num_pages[socket_id];
unsigned int num_pages_alloc;
if (num_pages == 0)
continue;
RTE_LOG(DEBUG, EAL,
"Allocating %u pages of size %" PRIu64 "M "
"on socket %i\n",
num_pages, hpi->hugepage_sz >> 20, socket_id);
/* we may not be able to allocate all pages in one go,
* because we break up our memory map into multiple
* memseg lists. therefore, try allocating multiple
* times and see if we can get the desired number of
* pages from multiple allocations.
*/
num_pages_alloc = 0;
do {
int i, cur_pages, needed;
needed = num_pages - num_pages_alloc;
pages = malloc(sizeof(*pages) * needed);
if (pages == NULL) {
RTE_LOG(ERR, EAL, "Failed to malloc pages\n");
return -1;
}
/* do not request exact number of pages */
cur_pages = eal_memalloc_alloc_seg_bulk(pages,
needed, hpi->hugepage_sz,
socket_id, false);
if (cur_pages <= 0) {
free(pages);
return -1;
}
/* mark preallocated pages as unfreeable */
for (i = 0; i < cur_pages; i++) {
struct rte_memseg *ms = pages[i];
ms->flags |=
RTE_MEMSEG_FLAG_DO_NOT_FREE;
}
free(pages);
num_pages_alloc += cur_pages;
} while (num_pages_alloc != num_pages);
}
}
/* if socket limits were specified, set them */
if (internal_conf->force_socket_limits) {
unsigned int i;
for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
uint64_t limit = internal_conf->socket_limit[i];
if (limit == 0)
continue;
if (rte_mem_alloc_validator_register("socket-limit",
limits_callback, i, limit))
RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n");
}
}
return 0;
}
__rte_unused /* function is unused on 32-bit builds */
static inline uint64_t
get_socket_mem_size(int socket)
{
uint64_t size = 0;
unsigned int i;
struct internal_config *internal_conf =
eal_get_internal_configuration();
for (i = 0; i < internal_conf->num_hugepage_sizes; i++) {
struct hugepage_info *hpi = &internal_conf->hugepage_info[i];
size += hpi->hugepage_sz * hpi->num_pages[socket];
}
return size;
}
int
eal_dynmem_calc_num_pages_per_socket(
uint64_t *memory, struct hugepage_info *hp_info,
struct hugepage_info *hp_used, unsigned int num_hp_info)
{
unsigned int socket, j, i = 0;
unsigned int requested, available;
int total_num_pages = 0;
uint64_t remaining_mem, cur_mem;
const struct internal_config *internal_conf =
eal_get_internal_configuration();
uint64_t total_mem = internal_conf->memory;
if (num_hp_info == 0)
return -1;
/* if specific memory amounts per socket weren't requested */
if (internal_conf->force_sockets == 0) {
size_t total_size;
#ifdef RTE_ARCH_64
int cpu_per_socket[RTE_MAX_NUMA_NODES];
size_t default_size;
unsigned int lcore_id;
/* Compute number of cores per socket */
memset(cpu_per_socket, 0, sizeof(cpu_per_socket));
RTE_LCORE_FOREACH(lcore_id) {
cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;
}
/*
* Automatically spread requested memory amongst detected
* sockets according to number of cores from CPU mask present
* on each socket.
*/
total_size = internal_conf->memory;
for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
socket++) {
/* Set memory amount per socket */
default_size = internal_conf->memory *
cpu_per_socket[socket] / rte_lcore_count();
/* Limit to maximum available memory on socket */
default_size = RTE_MIN(
default_size, get_socket_mem_size(socket));
/* Update sizes */
memory[socket] = default_size;
total_size -= default_size;
}
/*
* If some memory is remaining, try to allocate it by getting
* all available memory from sockets, one after the other.
*/
for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
socket++) {
/* take whatever is available */
default_size = RTE_MIN(
get_socket_mem_size(socket) - memory[socket],
total_size);
/* Update sizes */
memory[socket] += default_size;
total_size -= default_size;
}
#else
/* in 32-bit mode, allocate all of the memory only on main
* lcore socket
*/
total_size = internal_conf->memory;
for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
socket++) {
struct rte_config *cfg = rte_eal_get_configuration();
unsigned int main_lcore_socket;
main_lcore_socket =
rte_lcore_to_socket_id(cfg->main_lcore);
if (main_lcore_socket != socket)
continue;
/* Update sizes */
memory[socket] = total_size;
break;
}
#endif
}
for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_mem != 0;
socket++) {
/* skips if the memory on specific socket wasn't requested */
for (i = 0; i < num_hp_info && memory[socket] != 0; i++) {
rte_strscpy(hp_used[i].hugedir, hp_info[i].hugedir,
sizeof(hp_used[i].hugedir));
hp_used[i].num_pages[socket] = RTE_MIN(
memory[socket] / hp_info[i].hugepage_sz,
hp_info[i].num_pages[socket]);
cur_mem = hp_used[i].num_pages[socket] *
hp_used[i].hugepage_sz;
memory[socket] -= cur_mem;
total_mem -= cur_mem;
total_num_pages += hp_used[i].num_pages[socket];
/* check if we have met all memory requests */
if (memory[socket] == 0)
break;
/* Check if we have any more pages left at this size,
* if so, move on to next size.
*/
if (hp_used[i].num_pages[socket] ==
hp_info[i].num_pages[socket])
continue;
/* At this point we know that there are more pages
* available that are bigger than the memory we want,
* so lets see if we can get enough from other page
* sizes.
*/
remaining_mem = 0;
for (j = i+1; j < num_hp_info; j++)
remaining_mem += hp_info[j].hugepage_sz *
hp_info[j].num_pages[socket];
/* Is there enough other memory?
* If not, allocate another page and quit.
*/
if (remaining_mem < memory[socket]) {
cur_mem = RTE_MIN(
memory[socket], hp_info[i].hugepage_sz);
memory[socket] -= cur_mem;
total_mem -= cur_mem;
hp_used[i].num_pages[socket]++;
total_num_pages++;
break; /* we are done with this socket*/
}
}
/* if we didn't satisfy all memory requirements per socket */
if (memory[socket] > 0 &&
internal_conf->socket_mem[socket] != 0) {
/* to prevent icc errors */
requested = (unsigned int)(
internal_conf->socket_mem[socket] / 0x100000);
available = requested -
((unsigned int)(memory[socket] / 0x100000));
RTE_LOG(ERR, EAL, "Not enough memory available on "
"socket %u! Requested: %uMB, available: %uMB\n",
socket, requested, available);
return -1;
}
}
/* if we didn't satisfy total memory requirements */
if (total_mem > 0) {
requested = (unsigned int)(internal_conf->memory / 0x100000);
available = requested - (unsigned int)(total_mem / 0x100000);
RTE_LOG(ERR, EAL, "Not enough memory available! "
"Requested: %uMB, available: %uMB\n",
requested, available);
return -1;
}
return total_num_pages;
}

54
Programs/hugePage/common/eal_common_errno.c Normal file
View File

@@ -0,0 +1,54 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
/* Use XSI-compliant portable version of strerror_r() */
#undef _GNU_SOURCE
#include <stdio.h>
#include <string.h>
#include "rte_per_lcore.h"
#include "rte_errno.h"
#ifdef RTE_EXEC_ENV_WINDOWS
#define strerror_r(errnum, buf, buflen) strerror_s(buf, buflen, errnum)
#endif
RTE_DEFINE_PER_LCORE(int, _rte_errno);
const char *
rte_strerror(int errnum)
{
/* BSD puts a colon in the "unknown error" messages, Linux doesn't */
#ifdef RTE_EXEC_ENV_FREEBSD
static const char *sep = ":";
#else
static const char *sep = "";
#endif
#define RETVAL_SZ 256
static RTE_DEFINE_PER_LCORE(char[RETVAL_SZ], retval);
char *ret = RTE_PER_LCORE(retval);
/* since some implementations of strerror_r throw an error
* themselves if errnum is too big, we handle that case here */
if (errnum >= RTE_MAX_ERRNO)
#ifdef RTE_EXEC_ENV_WINDOWS
snprintf(ret, RETVAL_SZ, "Unknown error");
#else
snprintf(ret, RETVAL_SZ, "Unknown error%s %d", sep, errnum);
#endif
else
switch (errnum){
case E_RTE_SECONDARY:
return "Invalid call in secondary process";
case E_RTE_NO_CONFIG:
return "Missing rte_config structure";
default:
if (strerror_r(errnum, ret, RETVAL_SZ) != 0)
snprintf(ret, RETVAL_SZ, "Unknown error%s %d",
sep, errnum);
}
return ret;
}

1501
Programs/hugePage/common/eal_common_fbarray.c Normal file
View File

File diff suppressed because it is too large Load Diff

74
Programs/hugePage/common/eal_common_hexdump.c Normal file
View File

@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include "rte_hexdump.h"
#include "rte_string_fns.h"
#define LINE_LEN 128
void
rte_hexdump(FILE *f, const char *title, const void *buf, unsigned int len)
{
unsigned int i, out, ofs;
const unsigned char *data = buf;
char line[LINE_LEN]; /* space needed 8+16*3+3+16 == 75 */
fprintf(f, "%s at [%p], len=%u\n",
title ? : " Dump data", data, len);
ofs = 0;
while (ofs < len) {
/* format the line in the buffer */
out = snprintf(line, LINE_LEN, "%08X:", ofs);
for (i = 0; i < 16; i++) {
if (ofs + i < len)
snprintf(line + out, LINE_LEN - out,
" %02X", (data[ofs + i] & 0xff));
else
strcpy(line + out, " ");
out += 3;
}
for (; i <= 16; i++)
out += snprintf(line + out, LINE_LEN - out, " | ");
for (i = 0; ofs < len && i < 16; i++, ofs++) {
unsigned char c = data[ofs];
if (c < ' ' || c > '~')
c = '.';
out += snprintf(line + out, LINE_LEN - out, "%c", c);
}
fprintf(f, "%s\n", line);
}
fflush(f);
}
void
rte_memdump(FILE *f, const char *title, const void *buf, unsigned int len)
{
unsigned int i, out;
const unsigned char *data = buf;
char line[LINE_LEN];
if (title)
fprintf(f, "%s: ", title);
line[0] = '\0';
for (i = 0, out = 0; i < len; i++) {
/* Make sure we do not overrun the line buffer length. */
if (out >= LINE_LEN - 4) {
fprintf(f, "%s", line);
out = 0;
line[out] = '\0';
}
out += snprintf(line + out, LINE_LEN - out, "%02x%s",
(data[i] & 0xff), ((i + 1) < len) ? ":" : "");
}
if (out > 0)
fprintf(f, "%s", line);
fprintf(f, "\n");
fflush(f);
}

22
Programs/hugePage/common/eal_common_hypervisor.c Normal file
View File

@@ -0,0 +1,22 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2017 Mellanox Technologies, Ltd
*/
#include "rte_hypervisor.h"
const char *
rte_hypervisor_get_name(enum rte_hypervisor id)
{
switch (id) {
case RTE_HYPERVISOR_NONE:
return "none";
case RTE_HYPERVISOR_KVM:
return "KVM";
case RTE_HYPERVISOR_HYPERV:
return "Hyper-V";
case RTE_HYPERVISOR_VMWARE:
return "VMware";
default:
return "unknown";
}
}

501
Programs/hugePage/common/eal_common_interrupts.c Normal file
View File

@@ -0,0 +1,501 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2021 Marvell.
*/
#include <stdlib.h>
#include <string.h>
#include "rte_errno.h"
#include "rte_interrupts.h"
#include "rte_log.h"
#include "rte_malloc.h"
#include "eal_interrupts.h"
/* Macros to check for valid interrupt handle */
#define CHECK_VALID_INTR_HANDLE(intr_handle) do { \
if (intr_handle == NULL) { \
RTE_LOG(DEBUG, EAL, "Interrupt instance unallocated\n"); \
rte_errno = EINVAL; \
goto fail; \
} \
} while (0)
#define RTE_INTR_INSTANCE_KNOWN_FLAGS (RTE_INTR_INSTANCE_F_PRIVATE \
| RTE_INTR_INSTANCE_F_SHARED \
)
#define RTE_INTR_INSTANCE_USES_RTE_MEMORY(flags) \
(!!(flags & RTE_INTR_INSTANCE_F_SHARED))
struct rte_intr_handle *rte_intr_instance_alloc(uint32_t flags)
{
struct rte_intr_handle *intr_handle;
bool uses_rte_memory;
/* Check the flag passed by user, it should be part of the
* defined flags.
*/
if ((flags & ~RTE_INTR_INSTANCE_KNOWN_FLAGS) != 0) {
RTE_LOG(DEBUG, EAL, "Invalid alloc flag passed 0x%x\n", flags);
rte_errno = EINVAL;
return NULL;
}
uses_rte_memory = RTE_INTR_INSTANCE_USES_RTE_MEMORY(flags);
if (uses_rte_memory)
intr_handle = rte_zmalloc(NULL, sizeof(*intr_handle), 0);
else
intr_handle = calloc(1, sizeof(*intr_handle));
if (intr_handle == NULL) {
RTE_LOG(ERR, EAL, "Failed to allocate intr_handle\n");
rte_errno = ENOMEM;
return NULL;
}
if (uses_rte_memory) {
intr_handle->efds = rte_zmalloc(NULL,
RTE_MAX_RXTX_INTR_VEC_ID * sizeof(int), 0);
} else {
intr_handle->efds = calloc(RTE_MAX_RXTX_INTR_VEC_ID,
sizeof(int));
}
if (intr_handle->efds == NULL) {
RTE_LOG(ERR, EAL, "Fail to allocate event fd list\n");
rte_errno = ENOMEM;
goto fail;
}
if (uses_rte_memory) {
intr_handle->elist = rte_zmalloc(NULL,
RTE_MAX_RXTX_INTR_VEC_ID * sizeof(struct rte_epoll_event),
0);
} else {
intr_handle->elist = calloc(RTE_MAX_RXTX_INTR_VEC_ID,
sizeof(struct rte_epoll_event));
}
if (intr_handle->elist == NULL) {
RTE_LOG(ERR, EAL, "fail to allocate event fd list\n");
rte_errno = ENOMEM;
goto fail;
}
intr_handle->alloc_flags = flags;
intr_handle->nb_intr = RTE_MAX_RXTX_INTR_VEC_ID;
return intr_handle;
fail:
if (uses_rte_memory) {
rte_free(intr_handle->efds);
rte_free(intr_handle);
} else {
free(intr_handle->efds);
free(intr_handle);
}
return NULL;
}
struct rte_intr_handle *rte_intr_instance_dup(const struct rte_intr_handle *src)
{
struct rte_intr_handle *intr_handle;
if (src == NULL) {
RTE_LOG(DEBUG, EAL, "Source interrupt instance unallocated\n");
rte_errno = EINVAL;
return NULL;
}
intr_handle = rte_intr_instance_alloc(src->alloc_flags);
if (intr_handle != NULL) {
intr_handle->fd = src->fd;
intr_handle->dev_fd = src->dev_fd;
intr_handle->type = src->type;
intr_handle->max_intr = src->max_intr;
intr_handle->nb_efd = src->nb_efd;
intr_handle->efd_counter_size = src->efd_counter_size;
memcpy(intr_handle->efds, src->efds, src->nb_intr);
memcpy(intr_handle->elist, src->elist, src->nb_intr);
}
return intr_handle;
}
int rte_intr_event_list_update(struct rte_intr_handle *intr_handle, int size)
{
struct rte_epoll_event *tmp_elist;
bool uses_rte_memory;
int *tmp_efds;
CHECK_VALID_INTR_HANDLE(intr_handle);
if (size == 0) {
RTE_LOG(DEBUG, EAL, "Size can't be zero\n");
rte_errno = EINVAL;
goto fail;
}
uses_rte_memory =
RTE_INTR_INSTANCE_USES_RTE_MEMORY(intr_handle->alloc_flags);
if (uses_rte_memory) {
tmp_efds = rte_realloc(intr_handle->efds, size * sizeof(int),
0);
} else {
tmp_efds = realloc(intr_handle->efds, size * sizeof(int));
}
if (tmp_efds == NULL) {
RTE_LOG(ERR, EAL, "Failed to realloc the efds list\n");
rte_errno = ENOMEM;
goto fail;
}
intr_handle->efds = tmp_efds;
if (uses_rte_memory) {
tmp_elist = rte_realloc(intr_handle->elist,
size * sizeof(struct rte_epoll_event), 0);
} else {
tmp_elist = realloc(intr_handle->elist,
size * sizeof(struct rte_epoll_event));
}
if (tmp_elist == NULL) {
RTE_LOG(ERR, EAL, "Failed to realloc the event list\n");
rte_errno = ENOMEM;
goto fail;
}
intr_handle->elist = tmp_elist;
intr_handle->nb_intr = size;
return 0;
fail:
return -rte_errno;
}
void rte_intr_instance_free(struct rte_intr_handle *intr_handle)
{
if (intr_handle == NULL)
return;
if (RTE_INTR_INSTANCE_USES_RTE_MEMORY(intr_handle->alloc_flags)) {
rte_free(intr_handle->efds);
rte_free(intr_handle->elist);
rte_free(intr_handle);
} else {
free(intr_handle->efds);
free(intr_handle->elist);
free(intr_handle);
}
}
int rte_intr_fd_set(struct rte_intr_handle *intr_handle, int fd)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
intr_handle->fd = fd;
return 0;
fail:
return -rte_errno;
}
int rte_intr_fd_get(const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->fd;
fail:
return -1;
}
int rte_intr_type_set(struct rte_intr_handle *intr_handle,
enum rte_intr_handle_type type)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
intr_handle->type = type;
return 0;
fail:
return -rte_errno;
}
enum rte_intr_handle_type rte_intr_type_get(
const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->type;
fail:
return RTE_INTR_HANDLE_UNKNOWN;
}
int rte_intr_dev_fd_set(struct rte_intr_handle *intr_handle, int fd)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
intr_handle->dev_fd = fd;
return 0;
fail:
return -rte_errno;
}
int rte_intr_dev_fd_get(const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->dev_fd;
fail:
return -1;
}
int rte_intr_max_intr_set(struct rte_intr_handle *intr_handle,
int max_intr)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (max_intr > intr_handle->nb_intr) {
RTE_LOG(DEBUG, EAL, "Maximum interrupt vector ID (%d) exceeds "
"the number of available events (%d)\n", max_intr,
intr_handle->nb_intr);
rte_errno = ERANGE;
goto fail;
}
intr_handle->max_intr = max_intr;
return 0;
fail:
return -rte_errno;
}
int rte_intr_max_intr_get(const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->max_intr;
fail:
return -rte_errno;
}
int rte_intr_nb_efd_set(struct rte_intr_handle *intr_handle, int nb_efd)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
intr_handle->nb_efd = nb_efd;
return 0;
fail:
return -rte_errno;
}
int rte_intr_nb_efd_get(const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->nb_efd;
fail:
return -rte_errno;
}
int rte_intr_nb_intr_get(const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->nb_intr;
fail:
return -rte_errno;
}
int rte_intr_efd_counter_size_set(struct rte_intr_handle *intr_handle,
uint8_t efd_counter_size)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
intr_handle->efd_counter_size = efd_counter_size;
return 0;
fail:
return -rte_errno;
}
int rte_intr_efd_counter_size_get(const struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->efd_counter_size;
fail:
return -rte_errno;
}
int rte_intr_efds_index_get(const struct rte_intr_handle *intr_handle,
int index)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (index >= intr_handle->nb_intr) {
RTE_LOG(DEBUG, EAL, "Invalid index %d, max limit %d\n", index,
intr_handle->nb_intr);
rte_errno = EINVAL;
goto fail;
}
return intr_handle->efds[index];
fail:
return -rte_errno;
}
int rte_intr_efds_index_set(struct rte_intr_handle *intr_handle,
int index, int fd)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (index >= intr_handle->nb_intr) {
RTE_LOG(DEBUG, EAL, "Invalid index %d, max limit %d\n", index,
intr_handle->nb_intr);
rte_errno = ERANGE;
goto fail;
}
intr_handle->efds[index] = fd;
return 0;
fail:
return -rte_errno;
}
struct rte_epoll_event *rte_intr_elist_index_get(
struct rte_intr_handle *intr_handle, int index)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (index >= intr_handle->nb_intr) {
RTE_LOG(DEBUG, EAL, "Invalid index %d, max limit %d\n", index,
intr_handle->nb_intr);
rte_errno = ERANGE;
goto fail;
}
return &intr_handle->elist[index];
fail:
return NULL;
}
int rte_intr_elist_index_set(struct rte_intr_handle *intr_handle,
int index, struct rte_epoll_event elist)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (index >= intr_handle->nb_intr) {
RTE_LOG(DEBUG, EAL, "Invalid index %d, max limit %d\n", index,
intr_handle->nb_intr);
rte_errno = ERANGE;
goto fail;
}
intr_handle->elist[index] = elist;
return 0;
fail:
return -rte_errno;
}
int rte_intr_vec_list_alloc(struct rte_intr_handle *intr_handle,
const char *name, int size)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
/* Vector list already allocated */
if (intr_handle->intr_vec != NULL)
return 0;
if (size > intr_handle->nb_intr) {
RTE_LOG(DEBUG, EAL, "Invalid size %d, max limit %d\n", size,
intr_handle->nb_intr);
rte_errno = ERANGE;
goto fail;
}
if (RTE_INTR_INSTANCE_USES_RTE_MEMORY(intr_handle->alloc_flags))
intr_handle->intr_vec = rte_zmalloc(name, size * sizeof(int), 0);
else
intr_handle->intr_vec = calloc(size, sizeof(int));
if (intr_handle->intr_vec == NULL) {
RTE_LOG(ERR, EAL, "Failed to allocate %d intr_vec\n", size);
rte_errno = ENOMEM;
goto fail;
}
intr_handle->vec_list_size = size;
return 0;
fail:
return -rte_errno;
}
int rte_intr_vec_list_index_get(const struct rte_intr_handle *intr_handle,
int index)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (index >= intr_handle->vec_list_size) {
RTE_LOG(DEBUG, EAL, "Index %d greater than vec list size %d\n",
index, intr_handle->vec_list_size);
rte_errno = ERANGE;
goto fail;
}
return intr_handle->intr_vec[index];
fail:
return -rte_errno;
}
int rte_intr_vec_list_index_set(struct rte_intr_handle *intr_handle,
int index, int vec)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
if (index >= intr_handle->vec_list_size) {
RTE_LOG(DEBUG, EAL, "Index %d greater than vec list size %d\n",
index, intr_handle->vec_list_size);
rte_errno = ERANGE;
goto fail;
}
intr_handle->intr_vec[index] = vec;
return 0;
fail:
return -rte_errno;
}
void rte_intr_vec_list_free(struct rte_intr_handle *intr_handle)
{
if (intr_handle == NULL)
return;
if (RTE_INTR_INSTANCE_USES_RTE_MEMORY(intr_handle->alloc_flags))
rte_free(intr_handle->intr_vec);
else
free(intr_handle->intr_vec);
intr_handle->intr_vec = NULL;
intr_handle->vec_list_size = 0;
}
void *rte_intr_instance_windows_handle_get(struct rte_intr_handle *intr_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
return intr_handle->windows_handle;
fail:
return NULL;
}
int rte_intr_instance_windows_handle_set(struct rte_intr_handle *intr_handle,
void *windows_handle)
{
CHECK_VALID_INTR_HANDLE(intr_handle);
intr_handle->windows_handle = windows_handle;
return 0;
fail:
return -rte_errno;
}

111
Programs/hugePage/common/eal_common_launch.c Normal file
View File

@@ -0,0 +1,111 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <errno.h>
#include "rte_launch.h"
#include "rte_eal_trace.h"
#include "rte_pause.h"
#include "rte_lcore.h"
#include "eal_private.h"
#include "eal_thread.h"
/*
* Wait until a lcore finished its job.
*/
int
rte_eal_wait_lcore(unsigned worker_id)
{
while (__atomic_load_n(&lcore_config[worker_id].state,
__ATOMIC_ACQUIRE) != WAIT)
rte_pause();
return lcore_config[worker_id].ret;
}
/*
* Send a message to a worker lcore identified by worker_id to call a
* function f with argument arg. Once the execution is done, the
* remote lcore switches to WAIT state.
*/
int
rte_eal_remote_launch(lcore_function_t *f, void *arg, unsigned int worker_id)
{
int rc = -EBUSY;
/* Check if the worker is in 'WAIT' state. Use acquire order
* since 'state' variable is used as the guard variable.
*/
if (__atomic_load_n(&lcore_config[worker_id].state,
__ATOMIC_ACQUIRE) != WAIT)
goto finish;
lcore_config[worker_id].arg = arg;
/* Ensure that all the memory operations are completed
* before the worker thread starts running the function.
* Use worker thread function as the guard variable.
*/
__atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
rc = eal_thread_wake_worker(worker_id);
finish:
rte_eal_trace_thread_remote_launch(f, arg, worker_id, rc);
return rc;
}
/*
* Check that every WORKER lcores are in WAIT state, then call
* rte_eal_remote_launch() for all of them. If call_main is true
* (set to CALL_MAIN), also call the function on the main lcore.
*/
int
rte_eal_mp_remote_launch(int (*f)(void *), void *arg,
enum rte_rmt_call_main_t call_main)
{
int lcore_id;
int main_lcore = rte_get_main_lcore();
/* check state of lcores */
RTE_LCORE_FOREACH_WORKER(lcore_id) {
if (lcore_config[lcore_id].state != WAIT)
return -EBUSY;
}
/* send messages to cores */
RTE_LCORE_FOREACH_WORKER(lcore_id) {
rte_eal_remote_launch(f, arg, lcore_id);
}
if (call_main == CALL_MAIN) {
lcore_config[main_lcore].ret = f(arg);
lcore_config[main_lcore].state = WAIT;
}
return 0;
}
/*
* Return the state of the lcore identified by worker_id.
*/
enum rte_lcore_state_t
rte_eal_get_lcore_state(unsigned lcore_id)
{
return lcore_config[lcore_id].state;
}
/*
* Do a rte_eal_wait_lcore() for every lcore. The return values are
* ignored.
*/
void
rte_eal_mp_wait_lcore(void)
{
unsigned lcore_id;
RTE_LCORE_FOREACH_WORKER(lcore_id) {
rte_eal_wait_lcore(lcore_id);
}
}

458
Programs/hugePage/common/eal_common_lcore.c Normal file
View File

@@ -0,0 +1,458 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdlib.h>
#include <string.h>
#include "rte_common.h"
#include "rte_branch_prediction.h"
#include "rte_errno.h"
#include "rte_lcore.h"
#include "rte_log.h"
#include "eal_private.h"
#include "eal_thread.h"
unsigned int rte_get_main_lcore(void)
{
return rte_eal_get_configuration()->main_lcore;
}
unsigned int rte_lcore_count(void)
{
return rte_eal_get_configuration()->lcore_count;
}
int rte_lcore_index(int lcore_id)
{
if (unlikely(lcore_id >= RTE_MAX_LCORE))
return -1;
if (lcore_id < 0) {
if (rte_lcore_id() == LCORE_ID_ANY)
return -1;
lcore_id = (int)rte_lcore_id();
}
return lcore_config[lcore_id].core_index;
}
int rte_lcore_to_cpu_id(int lcore_id)
{
if (unlikely(lcore_id >= RTE_MAX_LCORE))
return -1;
if (lcore_id < 0) {
if (rte_lcore_id() == LCORE_ID_ANY)
return -1;
lcore_id = (int)rte_lcore_id();
}
return lcore_config[lcore_id].core_id;
}
rte_cpuset_t rte_lcore_cpuset(unsigned int lcore_id)
{
return lcore_config[lcore_id].cpuset;
}
enum rte_lcore_role_t
rte_eal_lcore_role(unsigned int lcore_id)
{
struct rte_config *cfg = rte_eal_get_configuration();
if (lcore_id >= RTE_MAX_LCORE)
return ROLE_OFF;
return cfg->lcore_role[lcore_id];
}
int
rte_lcore_has_role(unsigned int lcore_id, enum rte_lcore_role_t role)
{
struct rte_config *cfg = rte_eal_get_configuration();
if (lcore_id >= RTE_MAX_LCORE)
return -EINVAL;
return cfg->lcore_role[lcore_id] == role;
}
int rte_lcore_is_enabled(unsigned int lcore_id)
{
struct rte_config *cfg = rte_eal_get_configuration();
if (lcore_id >= RTE_MAX_LCORE)
return 0;
return cfg->lcore_role[lcore_id] == ROLE_RTE;
}
unsigned int rte_get_next_lcore(unsigned int i, int skip_main, int wrap)
{
i++;
if (wrap)
i %= RTE_MAX_LCORE;
while (i < RTE_MAX_LCORE) {
if (!rte_lcore_is_enabled(i) ||
(skip_main && (i == rte_get_main_lcore()))) {
i++;
if (wrap)
i %= RTE_MAX_LCORE;
continue;
}
break;
}
return i;
}
unsigned int
rte_lcore_to_socket_id(unsigned int lcore_id)
{
return lcore_config[lcore_id].socket_id;
}
static int
socket_id_cmp(const void *a, const void *b)
{
const int *lcore_id_a = a;
const int *lcore_id_b = b;
if (*lcore_id_a < *lcore_id_b)
return -1;
if (*lcore_id_a > *lcore_id_b)
return 1;
return 0;
}
/*
* Parse /sys/devices/system/cpu to get the number of physical and logical
* processors on the machine. The function will fill the cpu_info
* structure.
*/
int
rte_eal_cpu_init(void)
{
/* pointer to global configuration */
struct rte_config *config = rte_eal_get_configuration();
unsigned lcore_id;
unsigned count = 0;
unsigned int socket_id, prev_socket_id;
int lcore_to_socket_id[RTE_MAX_LCORE];
/*
* Parse the maximum set of logical cores, detect the subset of running
* ones and enable them by default.
*/
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
lcore_config[lcore_id].core_index = count;
/* init cpuset for per lcore config */
CPU_ZERO(&lcore_config[lcore_id].cpuset);
/* find socket first */
socket_id = eal_cpu_socket_id(lcore_id);
lcore_to_socket_id[lcore_id] = socket_id;
if (eal_cpu_detected(lcore_id) == 0) {
config->lcore_role[lcore_id] = ROLE_OFF;
lcore_config[lcore_id].core_index = -1;
continue;
}
/* By default, lcore 1:1 map to cpu id */
CPU_SET(lcore_id, &lcore_config[lcore_id].cpuset);
/* By default, each detected core is enabled */
config->lcore_role[lcore_id] = ROLE_RTE;
lcore_config[lcore_id].core_role = ROLE_RTE;
lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
lcore_config[lcore_id].socket_id = socket_id;
RTE_LOG(DEBUG, EAL, "Detected lcore %u as "
"core %u on socket %u\n",
lcore_id, lcore_config[lcore_id].core_id,
lcore_config[lcore_id].socket_id);
count++;
}
for (; lcore_id < CPU_SETSIZE; lcore_id++) {
if (eal_cpu_detected(lcore_id) == 0)
continue;
RTE_LOG(DEBUG, EAL, "Skipped lcore %u as core %u on socket %u\n",
lcore_id, eal_cpu_core_id(lcore_id),
eal_cpu_socket_id(lcore_id));
}
/* Set the count of enabled logical cores of the EAL configuration */
config->lcore_count = count;
RTE_LOG(DEBUG, EAL,
"Maximum logical cores by configuration: %u\n",
RTE_MAX_LCORE);
RTE_LOG(INFO, EAL, "Detected CPU lcores: %u\n", config->lcore_count);
/* sort all socket id's in ascending order */
qsort(lcore_to_socket_id, RTE_DIM(lcore_to_socket_id),
sizeof(lcore_to_socket_id[0]), socket_id_cmp);
prev_socket_id = -1;
config->numa_node_count = 0;
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
socket_id = lcore_to_socket_id[lcore_id];
if (socket_id != prev_socket_id)
config->numa_nodes[config->numa_node_count++] =
socket_id;
prev_socket_id = socket_id;
}
RTE_LOG(INFO, EAL, "Detected NUMA nodes: %u\n", config->numa_node_count);
return 0;
}
unsigned int
rte_socket_count(void)
{
const struct rte_config *config = rte_eal_get_configuration();
return config->numa_node_count;
}
int
rte_socket_id_by_idx(unsigned int idx)
{
const struct rte_config *config = rte_eal_get_configuration();
if (idx >= config->numa_node_count) {
rte_errno = EINVAL;
return -1;
}
return config->numa_nodes[idx];
}
static rte_rwlock_t lcore_lock = RTE_RWLOCK_INITIALIZER;
struct lcore_callback {
TAILQ_ENTRY(lcore_callback) next;
char *name;
rte_lcore_init_cb init;
rte_lcore_uninit_cb uninit;
void *arg;
};
static TAILQ_HEAD(lcore_callbacks_head, lcore_callback) lcore_callbacks =
TAILQ_HEAD_INITIALIZER(lcore_callbacks);
static int
callback_init(struct lcore_callback *callback, unsigned int lcore_id)
{
if (callback->init == NULL)
return 0;
RTE_LOG(DEBUG, EAL, "Call init for lcore callback %s, lcore_id %u\n",
callback->name, lcore_id);
return callback->init(lcore_id, callback->arg);
}
static void
callback_uninit(struct lcore_callback *callback, unsigned int lcore_id)
{
if (callback->uninit == NULL)
return;
RTE_LOG(DEBUG, EAL, "Call uninit for lcore callback %s, lcore_id %u\n",
callback->name, lcore_id);
callback->uninit(lcore_id, callback->arg);
}
static void
free_callback(struct lcore_callback *callback)
{
free(callback->name);
free(callback);
}
void *
rte_lcore_callback_register(const char *name, rte_lcore_init_cb init,
rte_lcore_uninit_cb uninit, void *arg)
{
struct rte_config *cfg = rte_eal_get_configuration();
struct lcore_callback *callback;
unsigned int lcore_id;
if (name == NULL)
return NULL;
callback = calloc(1, sizeof(*callback));
if (callback == NULL)
return NULL;
if (asprintf(&callback->name, "%s-%p", name, arg) == -1) {
free(callback);
return NULL;
}
callback->init = init;
callback->uninit = uninit;
callback->arg = arg;
rte_rwlock_write_lock(&lcore_lock);
if (callback->init == NULL)
goto no_init;
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (cfg->lcore_role[lcore_id] == ROLE_OFF)
continue;
if (callback_init(callback, lcore_id) == 0)
continue;
/* Callback refused init for this lcore, uninitialize all
* previous lcore.
*/
while (lcore_id-- != 0) {
if (cfg->lcore_role[lcore_id] == ROLE_OFF)
continue;
callback_uninit(callback, lcore_id);
}
free_callback(callback);
callback = NULL;
goto out;
}
no_init:
TAILQ_INSERT_TAIL(&lcore_callbacks, callback, next);
RTE_LOG(DEBUG, EAL, "Registered new lcore callback %s (%sinit, %suninit).\n",
callback->name, callback->init == NULL ? "NO " : "",
callback->uninit == NULL ? "NO " : "");
out:
rte_rwlock_write_unlock(&lcore_lock);
return callback;
}
void
rte_lcore_callback_unregister(void *handle)
{
struct rte_config *cfg = rte_eal_get_configuration();
struct lcore_callback *callback = handle;
unsigned int lcore_id;
if (callback == NULL)
return;
rte_rwlock_write_lock(&lcore_lock);
if (callback->uninit == NULL)
goto no_uninit;
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (cfg->lcore_role[lcore_id] == ROLE_OFF)
continue;
callback_uninit(callback, lcore_id);
}
no_uninit:
TAILQ_REMOVE(&lcore_callbacks, callback, next);
rte_rwlock_write_unlock(&lcore_lock);
RTE_LOG(DEBUG, EAL, "Unregistered lcore callback %s-%p.\n",
callback->name, callback->arg);
free_callback(callback);
}
unsigned int
eal_lcore_non_eal_allocate(void)
{
struct rte_config *cfg = rte_eal_get_configuration();
struct lcore_callback *callback;
struct lcore_callback *prev;
unsigned int lcore_id;
rte_rwlock_write_lock(&lcore_lock);
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (cfg->lcore_role[lcore_id] != ROLE_OFF)
continue;
cfg->lcore_role[lcore_id] = ROLE_NON_EAL;
cfg->lcore_count++;
break;
}
if (lcore_id == RTE_MAX_LCORE) {
RTE_LOG(DEBUG, EAL, "No lcore available.\n");
goto out;
}
TAILQ_FOREACH(callback, &lcore_callbacks, next) {
if (callback_init(callback, lcore_id) == 0)
continue;
/* Callback refused init for this lcore, call uninit for all
* previous callbacks.
*/
prev = TAILQ_PREV(callback, lcore_callbacks_head, next);
while (prev != NULL) {
callback_uninit(prev, lcore_id);
prev = TAILQ_PREV(prev, lcore_callbacks_head, next);
}
RTE_LOG(DEBUG, EAL, "Initialization refused for lcore %u.\n",
lcore_id);
cfg->lcore_role[lcore_id] = ROLE_OFF;
cfg->lcore_count--;
lcore_id = RTE_MAX_LCORE;
goto out;
}
out:
rte_rwlock_write_unlock(&lcore_lock);
return lcore_id;
}
void
eal_lcore_non_eal_release(unsigned int lcore_id)
{
struct rte_config *cfg = rte_eal_get_configuration();
struct lcore_callback *callback;
rte_rwlock_write_lock(&lcore_lock);
if (cfg->lcore_role[lcore_id] != ROLE_NON_EAL)
goto out;
TAILQ_FOREACH(callback, &lcore_callbacks, next)
callback_uninit(callback, lcore_id);
cfg->lcore_role[lcore_id] = ROLE_OFF;
cfg->lcore_count--;
out:
rte_rwlock_write_unlock(&lcore_lock);
}
int
rte_lcore_iterate(rte_lcore_iterate_cb cb, void *arg)
{
struct rte_config *cfg = rte_eal_get_configuration();
unsigned int lcore_id;
int ret = 0;
rte_rwlock_read_lock(&lcore_lock);
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (cfg->lcore_role[lcore_id] == ROLE_OFF)
continue;
ret = cb(lcore_id, arg);
if (ret != 0)
break;
}
rte_rwlock_read_unlock(&lcore_lock);
return ret;
}
static int
lcore_dump_cb(unsigned int lcore_id, void *arg)
{
struct rte_config *cfg = rte_eal_get_configuration();
char cpuset[RTE_CPU_AFFINITY_STR_LEN];
const char *role;
FILE *f = arg;
int ret;
switch (cfg->lcore_role[lcore_id]) {
case ROLE_RTE:
role = "RTE";
break;
case ROLE_SERVICE:
role = "SERVICE";
break;
case ROLE_NON_EAL:
role = "NON_EAL";
break;
default:
role = "UNKNOWN";
break;
}
ret = eal_thread_dump_affinity(&lcore_config[lcore_id].cpuset, cpuset,
sizeof(cpuset));
fprintf(f, "lcore %u, socket %u, role %s, cpuset %s%s\n", lcore_id,
rte_lcore_to_socket_id(lcore_id), role, cpuset,
ret == 0 ? "" : "...");
return 0;
}
void
rte_lcore_dump(FILE *f)
{
rte_lcore_iterate(lcore_dump_cb, f);
}

548
Programs/hugePage/common/eal_common_log.c Normal file
View File

@@ -0,0 +1,548 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include <stdint.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <regex.h>
#include <fnmatch.h>
#include <sys/queue.h>
#include "rte_log.h"
#include "rte_per_lcore.h"
#include "eal_log.h"
#include "eal_private.h"
struct rte_log_dynamic_type {
const char *name;
uint32_t loglevel;
};
/** The rte_log structure. */
static struct rte_logs {
uint32_t type; /**< Bitfield with enabled logs. */
uint32_t level; /**< Log level. */
FILE *file; /**< Output file set by rte_openlog_stream, or NULL. */
size_t dynamic_types_len;
struct rte_log_dynamic_type *dynamic_types;
} rte_logs = {
.type = UINT32_MAX,
.level = RTE_LOG_DEBUG,
};
struct rte_eal_opt_loglevel {
/** Next list entry */
TAILQ_ENTRY(rte_eal_opt_loglevel) next;
/** Compiled regular expression obtained from the option */
regex_t re_match;
/** Globbing pattern option */
char *pattern;
/** Log level value obtained from the option */
uint32_t level;
};
TAILQ_HEAD(rte_eal_opt_loglevel_list, rte_eal_opt_loglevel);
/** List of valid EAL log level options */
static struct rte_eal_opt_loglevel_list opt_loglevel_list =
TAILQ_HEAD_INITIALIZER(opt_loglevel_list);
/* Stream to use for logging if rte_logs.file is NULL */
static FILE *default_log_stream;
/**
* This global structure stores some information about the message
* that is currently being processed by one lcore
*/
struct log_cur_msg {
uint32_t loglevel; /**< log level - see rte_log.h */
uint32_t logtype; /**< log type - see rte_log.h */
};
/* per core log */
static RTE_DEFINE_PER_LCORE(struct log_cur_msg, log_cur_msg);
/* default logs */
/* Change the stream that will be used by logging system */
int
rte_openlog_stream(FILE *f)
{
rte_logs.file = f;
return 0;
}
FILE *
rte_log_get_stream(void)
{
FILE *f = rte_logs.file;
if (f == NULL) {
/*
* Grab the current value of stderr here, rather than
* just initializing default_log_stream to stderr. This
* ensures that we will always use the current value
* of stderr, even if the application closes and
* reopens it.
*/
return default_log_stream ? : stderr;
}
return f;
}
/* Set global log level */
void
rte_log_set_global_level(uint32_t level)
{
rte_logs.level = (uint32_t)level;
}
/* Get global log level */
uint32_t
rte_log_get_global_level(void)
{
return rte_logs.level;
}
int
rte_log_get_level(uint32_t type)
{
if (type >= rte_logs.dynamic_types_len)
return -1;
return rte_logs.dynamic_types[type].loglevel;
}
bool
rte_log_can_log(uint32_t logtype, uint32_t level)
{
int log_level;
if (level > rte_log_get_global_level())
return false;
log_level = rte_log_get_level(logtype);
if (log_level < 0)
return false;
if (level > (uint32_t)log_level)
return false;
return true;
}
static void
logtype_set_level(uint32_t type, uint32_t level)
{
uint32_t current = rte_logs.dynamic_types[type].loglevel;
if (current != level) {
rte_logs.dynamic_types[type].loglevel = level;
RTE_LOG(DEBUG, EAL, "%s log level changed from %s to %s\n",
rte_logs.dynamic_types[type].name == NULL ?
"" : rte_logs.dynamic_types[type].name,
eal_log_level2str(current),
eal_log_level2str(level));
}
}
int
rte_log_set_level(uint32_t type, uint32_t level)
{
if (type >= rte_logs.dynamic_types_len)
return -1;
if (level > RTE_LOG_MAX)
return -1;
logtype_set_level(type, level);
return 0;
}
/* set log level by regular expression */
int
rte_log_set_level_regexp(const char *regex, uint32_t level)
{
regex_t r;
size_t i;
if (level > RTE_LOG_MAX)
return -1;
if (regcomp(&r, regex, 0) != 0)
return -1;
for (i = 0; i < rte_logs.dynamic_types_len; i++) {
if (rte_logs.dynamic_types[i].name == NULL)
continue;
if (regexec(&r, rte_logs.dynamic_types[i].name, 0,
NULL, 0) == 0)
logtype_set_level(i, level);
}
regfree(&r);
return 0;
}
/*
* Save the type string and the loglevel for later dynamic
* logtypes which may register later.
*/
static int
log_save_level(uint32_t priority, const char *regex, const char *pattern)
{
struct rte_eal_opt_loglevel *opt_ll = NULL;
opt_ll = malloc(sizeof(*opt_ll));
if (opt_ll == NULL)
goto fail;
opt_ll->level = priority;
if (regex) {
opt_ll->pattern = NULL;
if (regcomp(&opt_ll->re_match, regex, 0) != 0)
goto fail;
} else if (pattern) {
opt_ll->pattern = strdup(pattern);
if (opt_ll->pattern == NULL)
goto fail;
} else
goto fail;
TAILQ_INSERT_HEAD(&opt_loglevel_list, opt_ll, next);
return 0;
fail:
free(opt_ll);
return -1;
}
int
eal_log_save_regexp(const char *regex, uint32_t level)
{
return log_save_level(level, regex, NULL);
}
/* set log level based on globbing pattern */
int
rte_log_set_level_pattern(const char *pattern, uint32_t level)
{
size_t i;
if (level > RTE_LOG_MAX)
return -1;
for (i = 0; i < rte_logs.dynamic_types_len; i++) {
if (rte_logs.dynamic_types[i].name == NULL)
continue;
if (fnmatch(pattern, rte_logs.dynamic_types[i].name, 0) == 0)
logtype_set_level(i, level);
}
return 0;
}
int
eal_log_save_pattern(const char *pattern, uint32_t level)
{
return log_save_level(level, NULL, pattern);
}
/* get the current loglevel for the message being processed */
int rte_log_cur_msg_loglevel(void)
{
return RTE_PER_LCORE(log_cur_msg).loglevel;
}
/* get the current logtype for the message being processed */
int rte_log_cur_msg_logtype(void)
{
return RTE_PER_LCORE(log_cur_msg).logtype;
}
static int
log_lookup(const char *name)
{
size_t i;
for (i = 0; i < rte_logs.dynamic_types_len; i++) {
if (rte_logs.dynamic_types[i].name == NULL)
continue;
if (strcmp(name, rte_logs.dynamic_types[i].name) == 0)
return i;
}
return -1;
}
static int
log_register(const char *name, uint32_t level)
{
struct rte_log_dynamic_type *new_dynamic_types;
int id;
id = log_lookup(name);
if (id >= 0)
return id;
new_dynamic_types = realloc(rte_logs.dynamic_types,
sizeof(struct rte_log_dynamic_type) *
(rte_logs.dynamic_types_len + 1));
if (new_dynamic_types == NULL)
return -ENOMEM;
rte_logs.dynamic_types = new_dynamic_types;
id = rte_logs.dynamic_types_len;
memset(&rte_logs.dynamic_types[id], 0,
sizeof(rte_logs.dynamic_types[id]));
rte_logs.dynamic_types[id].name = strdup(name);
if (rte_logs.dynamic_types[id].name == NULL)
return -ENOMEM;
logtype_set_level(id, level);
rte_logs.dynamic_types_len++;
return id;
}
/* register an extended log type */
int
rte_log_register(const char *name)
{
return log_register(name, RTE_LOG_INFO);
}
/* Register an extended log type and try to pick its level from EAL options */
int
rte_log_register_type_and_pick_level(const char *name, uint32_t level_def)
{
struct rte_eal_opt_loglevel *opt_ll;
uint32_t level = level_def;
TAILQ_FOREACH(opt_ll, &opt_loglevel_list, next) {
if (opt_ll->level > RTE_LOG_MAX)
continue;
if (opt_ll->pattern) {
if (fnmatch(opt_ll->pattern, name, 0) == 0)
level = opt_ll->level;
} else {
if (regexec(&opt_ll->re_match, name, 0, NULL, 0) == 0)
level = opt_ll->level;
}
}
return log_register(name, level);
}
struct logtype {
uint32_t log_id;
const char *logtype;
};
static const struct logtype logtype_strings[] = {
{RTE_LOGTYPE_EAL, "lib.eal"},
{RTE_LOGTYPE_MALLOC, "lib.malloc"},
{RTE_LOGTYPE_RING, "lib.ring"},
{RTE_LOGTYPE_MEMPOOL, "lib.mempool"},
{RTE_LOGTYPE_TIMER, "lib.timer"},
{RTE_LOGTYPE_PMD, "pmd"},
{RTE_LOGTYPE_HASH, "lib.hash"},
{RTE_LOGTYPE_LPM, "lib.lpm"},
{RTE_LOGTYPE_KNI, "lib.kni"},
{RTE_LOGTYPE_ACL, "lib.acl"},
{RTE_LOGTYPE_POWER, "lib.power"},
{RTE_LOGTYPE_METER, "lib.meter"},
{RTE_LOGTYPE_SCHED, "lib.sched"},
{RTE_LOGTYPE_PORT, "lib.port"},
{RTE_LOGTYPE_TABLE, "lib.table"},
{RTE_LOGTYPE_PIPELINE, "lib.pipeline"},
{RTE_LOGTYPE_MBUF, "lib.mbuf"},
{RTE_LOGTYPE_CRYPTODEV, "lib.cryptodev"},
{RTE_LOGTYPE_EFD, "lib.efd"},
{RTE_LOGTYPE_EVENTDEV, "lib.eventdev"},
{RTE_LOGTYPE_GSO, "lib.gso"},
{RTE_LOGTYPE_USER1, "user1"},
{RTE_LOGTYPE_USER2, "user2"},
{RTE_LOGTYPE_USER3, "user3"},
{RTE_LOGTYPE_USER4, "user4"},
{RTE_LOGTYPE_USER5, "user5"},
{RTE_LOGTYPE_USER6, "user6"},
{RTE_LOGTYPE_USER7, "user7"},
{RTE_LOGTYPE_USER8, "user8"}
};
/* Logging should be first initializer (before drivers and bus) */
RTE_INIT_PRIO(log_init, LOG)
{
uint32_t i;
rte_log_set_global_level(RTE_LOG_DEBUG);
rte_logs.dynamic_types = calloc(RTE_LOGTYPE_FIRST_EXT_ID,
sizeof(struct rte_log_dynamic_type));
if (rte_logs.dynamic_types == NULL)
return;
/* register legacy log types */
for (i = 0; i < RTE_DIM(logtype_strings); i++) {
rte_logs.dynamic_types[logtype_strings[i].log_id].name =
strdup(logtype_strings[i].logtype);
logtype_set_level(logtype_strings[i].log_id, RTE_LOG_INFO);
}
rte_logs.dynamic_types_len = RTE_LOGTYPE_FIRST_EXT_ID;
}
const char *
eal_log_level2str(uint32_t level)
{
switch (level) {
case 0: return "disabled";
case RTE_LOG_EMERG: return "emergency";
case RTE_LOG_ALERT: return "alert";
case RTE_LOG_CRIT: return "critical";
case RTE_LOG_ERR: return "error";
case RTE_LOG_WARNING: return "warning";
case RTE_LOG_NOTICE: return "notice";
case RTE_LOG_INFO: return "info";
case RTE_LOG_DEBUG: return "debug";
default: return "unknown";
}
}
static int
log_type_compare(const void *a, const void *b)
{
const struct rte_log_dynamic_type *type_a = a;
const struct rte_log_dynamic_type *type_b = b;
if (type_a->name == NULL && type_b->name == NULL)
return 0;
if (type_a->name == NULL)
return -1;
if (type_b->name == NULL)
return 1;
return strcmp(type_a->name, type_b->name);
}
/* Dump name of each logtype, one per line. */
void
rte_log_list_types(FILE *out, const char *prefix)
{
struct rte_log_dynamic_type *sorted_types;
const size_t type_size = sizeof(rte_logs.dynamic_types[0]);
const size_t type_count = rte_logs.dynamic_types_len;
const size_t total_size = type_size * type_count;
size_t type;
sorted_types = malloc(total_size);
if (sorted_types == NULL) {
/* no sorting - unlikely */
sorted_types = rte_logs.dynamic_types;
} else {
memcpy(sorted_types, rte_logs.dynamic_types, total_size);
qsort(sorted_types, type_count, type_size, log_type_compare);
}
for (type = 0; type < type_count; ++type) {
if (sorted_types[type].name == NULL)
continue;
fprintf(out, "%s%s\n", prefix, sorted_types[type].name);
}
if (sorted_types != rte_logs.dynamic_types)
free(sorted_types);
}
/* dump global level and registered log types */
void
rte_log_dump(FILE *f)
{
size_t i;
fprintf(f, "global log level is %s\n",
eal_log_level2str(rte_log_get_global_level()));
for (i = 0; i < rte_logs.dynamic_types_len; i++) {
if (rte_logs.dynamic_types[i].name == NULL)
continue;
fprintf(f, "id %zu: %s, level is %s\n",
i, rte_logs.dynamic_types[i].name,
eal_log_level2str(rte_logs.dynamic_types[i].loglevel));
}
}
/*
* Generates a log message The message will be sent in the stream
* defined by the previous call to rte_openlog_stream().
*/
int
rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap)
{
FILE *f = rte_log_get_stream();
int ret;
if (logtype >= rte_logs.dynamic_types_len)
return -1;
if (!rte_log_can_log(logtype, level))
return 0;
/* save loglevel and logtype in a global per-lcore variable */
RTE_PER_LCORE(log_cur_msg).loglevel = level;
RTE_PER_LCORE(log_cur_msg).logtype = logtype;
ret = vfprintf(f, format, ap);
fflush(f);
return ret;
}
/*
* Generates a log message The message will be sent in the stream
* defined by the previous call to rte_openlog_stream().
* No need to check level here, done by rte_vlog().
*/
int
rte_log(uint32_t level, uint32_t logtype, const char *format, ...)
{
va_list ap;
int ret;
va_start(ap, format);
ret = rte_vlog(level, logtype, format, ap);
va_end(ap);
return ret;
}
/*
* Called by environment-specific initialization functions.
*/
void
eal_log_set_default(FILE *default_log)
{
default_log_stream = default_log;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
RTE_LOG(NOTICE, EAL,
"Debug dataplane logs available - lower performance\n");
#endif
}
/*
* Called by eal_cleanup
*/
void
rte_eal_log_cleanup(void)
{
if (default_log_stream) {
fclose(default_log_stream);
default_log_stream = NULL;
}
}

175
Programs/hugePage/common/eal_common_mcfg.c Normal file
View File

@@ -0,0 +1,175 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2019 Intel Corporation
*/
#include "rte_eal_memconfig.h"
#include "rte_version.h"
#include "eal_internal_cfg.h"
#include "eal_memcfg.h"
#include "eal_private.h"
void
eal_mcfg_complete(void)
{
struct rte_config *cfg = rte_eal_get_configuration();
struct rte_mem_config *mcfg = cfg->mem_config;
struct internal_config *internal_conf =
eal_get_internal_configuration();
/* ALL shared mem_config related INIT DONE */
if (cfg->process_type == RTE_PROC_PRIMARY)
mcfg->magic = RTE_MAGIC;
internal_conf->init_complete = 1;
}
void
eal_mcfg_wait_complete(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
/* wait until shared mem_config finish initialising */
rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, __ATOMIC_RELAXED);
}
int
eal_mcfg_check_version(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
/* check if version from memconfig matches compiled in macro */
if (mcfg->version != RTE_VERSION)
return -1;
return 0;
}
void
eal_mcfg_update_internal(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
struct internal_config *internal_conf =
eal_get_internal_configuration();
internal_conf->legacy_mem = mcfg->legacy_mem;
internal_conf->single_file_segments = mcfg->single_file_segments;
}
void
eal_mcfg_update_from_internal(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
const struct internal_config *internal_conf =
eal_get_internal_configuration();
mcfg->legacy_mem = internal_conf->legacy_mem;
mcfg->single_file_segments = internal_conf->single_file_segments;
/* record current DPDK version */
mcfg->version = RTE_VERSION;
}
void
rte_mcfg_mem_read_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
}
void
rte_mcfg_mem_read_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
}
void
rte_mcfg_mem_write_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_lock(&mcfg->memory_hotplug_lock);
}
void
rte_mcfg_mem_write_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock);
}
void
rte_mcfg_tailq_read_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_lock(&mcfg->qlock);
}
void
rte_mcfg_tailq_read_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_unlock(&mcfg->qlock);
}
void
rte_mcfg_tailq_write_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_lock(&mcfg->qlock);
}
void
rte_mcfg_tailq_write_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_unlock(&mcfg->qlock);
}
void
rte_mcfg_mempool_read_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_lock(&mcfg->mplock);
}
void
rte_mcfg_mempool_read_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_unlock(&mcfg->mplock);
}
void
rte_mcfg_mempool_write_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_lock(&mcfg->mplock);
}
void
rte_mcfg_mempool_write_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_unlock(&mcfg->mplock);
}
void
rte_mcfg_timer_lock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_spinlock_lock(&mcfg->tlock);
}
void
rte_mcfg_timer_unlock(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
rte_spinlock_unlock(&mcfg->tlock);
}
bool
rte_mcfg_get_single_file_segments(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
return (bool)mcfg->single_file_segments;
}

363
Programs/hugePage/common/eal_common_memalloc.c Normal file
View File

@@ -0,0 +1,363 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017-2018 Intel Corporation
*/
#include <stdlib.h>
#include <string.h>
#include "rte_errno.h>
#include "rte_fbarray.h>
#include "rte_memory.h>
#include "rte_string_fns.h>
#include "eal_private.h"
#include "eal_internal_cfg.h"
#include "eal_memalloc.h"
struct mem_event_callback_entry {
TAILQ_ENTRY(mem_event_callback_entry) next;
char name[RTE_MEM_EVENT_CALLBACK_NAME_LEN];
rte_mem_event_callback_t clb;
void *arg;
};
struct mem_alloc_validator_entry {
TAILQ_ENTRY(mem_alloc_validator_entry) next;
char name[RTE_MEM_ALLOC_VALIDATOR_NAME_LEN];
rte_mem_alloc_validator_t clb;
int socket_id;
size_t limit;
};
/** Double linked list of actions. */
TAILQ_HEAD(mem_event_callback_entry_list, mem_event_callback_entry);
TAILQ_HEAD(mem_alloc_validator_entry_list, mem_alloc_validator_entry);
static struct mem_event_callback_entry_list mem_event_callback_list =
TAILQ_HEAD_INITIALIZER(mem_event_callback_list);
static rte_rwlock_t mem_event_rwlock = RTE_RWLOCK_INITIALIZER;
static struct mem_alloc_validator_entry_list mem_alloc_validator_list =
TAILQ_HEAD_INITIALIZER(mem_alloc_validator_list);
static rte_rwlock_t mem_alloc_validator_rwlock = RTE_RWLOCK_INITIALIZER;
static struct mem_event_callback_entry *
find_mem_event_callback(const char *name, void *arg)
{
struct mem_event_callback_entry *r;
TAILQ_FOREACH(r, &mem_event_callback_list, next) {
if (!strcmp(r->name, name) && r->arg == arg)
break;
}
return r;
}
static struct mem_alloc_validator_entry *
find_mem_alloc_validator(const char *name, int socket_id)
{
struct mem_alloc_validator_entry *r;
TAILQ_FOREACH(r, &mem_alloc_validator_list, next) {
if (!strcmp(r->name, name) && r->socket_id == socket_id)
break;
}
return r;
}
bool
eal_memalloc_is_contig(const struct rte_memseg_list *msl, void *start,
size_t len)
{
void *end, *aligned_start, *aligned_end;
size_t pgsz = (size_t)msl->page_sz;
const struct rte_memseg *ms;
const struct internal_config *internal_conf =
eal_get_internal_configuration();
/* for IOVA_VA, it's always contiguous */
if (rte_eal_iova_mode() == RTE_IOVA_VA && !msl->external)
return true;
/* for legacy memory, it's always contiguous */
if (internal_conf->legacy_mem)
return true;
end = RTE_PTR_ADD(start, len);
/* for nohuge, we check pagemap, otherwise check memseg */
if (!rte_eal_has_hugepages()) {
rte_iova_t cur, expected;
aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz);
aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz);
/* if start and end are on the same page, bail out early */
if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz)
return true;
/* skip first iteration */
cur = rte_mem_virt2iova(aligned_start);
expected = cur + pgsz;
aligned_start = RTE_PTR_ADD(aligned_start, pgsz);
while (aligned_start < aligned_end) {
cur = rte_mem_virt2iova(aligned_start);
if (cur != expected)
return false;
aligned_start = RTE_PTR_ADD(aligned_start, pgsz);
expected += pgsz;
}
} else {
int start_seg, end_seg, cur_seg;
rte_iova_t cur, expected;
aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz);
aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz);
start_seg = RTE_PTR_DIFF(aligned_start, msl->base_va) /
pgsz;
end_seg = RTE_PTR_DIFF(aligned_end, msl->base_va) /
pgsz;
/* if start and end are on the same page, bail out early */
if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz)
return true;
/* skip first iteration */
ms = rte_fbarray_get(&msl->memseg_arr, start_seg);
cur = ms->iova;
expected = cur + pgsz;
/* if we can't access IOVA addresses, assume non-contiguous */
if (cur == RTE_BAD_IOVA)
return false;
for (cur_seg = start_seg + 1; cur_seg < end_seg;
cur_seg++, expected += pgsz) {
ms = rte_fbarray_get(&msl->memseg_arr, cur_seg);
if (ms->iova != expected)
return false;
}
}
return true;
}
int
eal_memalloc_mem_event_callback_register(const char *name,
rte_mem_event_callback_t clb, void *arg)
{
struct mem_event_callback_entry *entry;
int ret, len;
if (name == NULL || clb == NULL) {
rte_errno = EINVAL;
return -1;
}
len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
if (len == 0) {
rte_errno = EINVAL;
return -1;
} else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
rte_errno = ENAMETOOLONG;
return -1;
}
rte_rwlock_write_lock(&mem_event_rwlock);
entry = find_mem_event_callback(name, arg);
if (entry != NULL) {
rte_errno = EEXIST;
ret = -1;
goto unlock;
}
entry = malloc(sizeof(*entry));
if (entry == NULL) {
rte_errno = ENOMEM;
ret = -1;
goto unlock;
}
/* callback successfully created and is valid, add it to the list */
entry->clb = clb;
entry->arg = arg;
strlcpy(entry->name, name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
TAILQ_INSERT_TAIL(&mem_event_callback_list, entry, next);
ret = 0;
RTE_LOG(DEBUG, EAL, "Mem event callback '%s:%p' registered\n",
name, arg);
unlock:
rte_rwlock_write_unlock(&mem_event_rwlock);
return ret;
}
int
eal_memalloc_mem_event_callback_unregister(const char *name, void *arg)
{
struct mem_event_callback_entry *entry;
int ret, len;
if (name == NULL) {
rte_errno = EINVAL;
return -1;
}
len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
if (len == 0) {
rte_errno = EINVAL;
return -1;
} else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
rte_errno = ENAMETOOLONG;
return -1;
}
rte_rwlock_write_lock(&mem_event_rwlock);
entry = find_mem_event_callback(name, arg);
if (entry == NULL) {
rte_errno = ENOENT;
ret = -1;
goto unlock;
}
TAILQ_REMOVE(&mem_event_callback_list, entry, next);
free(entry);
ret = 0;
RTE_LOG(DEBUG, EAL, "Mem event callback '%s:%p' unregistered\n",
name, arg);
unlock:
rte_rwlock_write_unlock(&mem_event_rwlock);
return ret;
}
void
eal_memalloc_mem_event_notify(enum rte_mem_event event, const void *start,
size_t len)
{
struct mem_event_callback_entry *entry;
rte_rwlock_read_lock(&mem_event_rwlock);
TAILQ_FOREACH(entry, &mem_event_callback_list, next) {
RTE_LOG(DEBUG, EAL, "Calling mem event callback '%s:%p'\n",
entry->name, entry->arg);
entry->clb(event, start, len, entry->arg);
}
rte_rwlock_read_unlock(&mem_event_rwlock);
}
int
eal_memalloc_mem_alloc_validator_register(const char *name,
rte_mem_alloc_validator_t clb, int socket_id, size_t limit)
{
struct mem_alloc_validator_entry *entry;
int ret, len;
if (name == NULL || clb == NULL || socket_id < 0) {
rte_errno = EINVAL;
return -1;
}
len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
if (len == 0) {
rte_errno = EINVAL;
return -1;
} else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) {
rte_errno = ENAMETOOLONG;
return -1;
}
rte_rwlock_write_lock(&mem_alloc_validator_rwlock);
entry = find_mem_alloc_validator(name, socket_id);
if (entry != NULL) {
rte_errno = EEXIST;
ret = -1;
goto unlock;
}
entry = malloc(sizeof(*entry));
if (entry == NULL) {
rte_errno = ENOMEM;
ret = -1;
goto unlock;
}
/* callback successfully created and is valid, add it to the list */
entry->clb = clb;
entry->socket_id = socket_id;
entry->limit = limit;
strlcpy(entry->name, name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
TAILQ_INSERT_TAIL(&mem_alloc_validator_list, entry, next);
ret = 0;
RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i with limit %zu registered\n",
name, socket_id, limit);
unlock:
rte_rwlock_write_unlock(&mem_alloc_validator_rwlock);
return ret;
}
int
eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id)
{
struct mem_alloc_validator_entry *entry;
int ret, len;
if (name == NULL || socket_id < 0) {
rte_errno = EINVAL;
return -1;
}
len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
if (len == 0) {
rte_errno = EINVAL;
return -1;
} else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) {
rte_errno = ENAMETOOLONG;
return -1;
}
rte_rwlock_write_lock(&mem_alloc_validator_rwlock);
entry = find_mem_alloc_validator(name, socket_id);
if (entry == NULL) {
rte_errno = ENOENT;
ret = -1;
goto unlock;
}
TAILQ_REMOVE(&mem_alloc_validator_list, entry, next);
free(entry);
ret = 0;
RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i unregistered\n",
name, socket_id);
unlock:
rte_rwlock_write_unlock(&mem_alloc_validator_rwlock);
return ret;
}
int
eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len)
{
struct mem_alloc_validator_entry *entry;
int ret = 0;
rte_rwlock_read_lock(&mem_alloc_validator_rwlock);
TAILQ_FOREACH(entry, &mem_alloc_validator_list, next) {
if (entry->socket_id != socket_id || entry->limit > new_len)
continue;
RTE_LOG(DEBUG, EAL, "Calling mem alloc validator '%s' on socket %i\n",
entry->name, entry->socket_id);
if (entry->clb(socket_id, entry->limit, new_len) < 0)
ret = -1;
}
rte_rwlock_read_unlock(&mem_alloc_validator_rwlock);
return ret;
}

1283
Programs/hugePage/common/eal_common_memory.c Normal file
View File

File diff suppressed because it is too large Load Diff

432
Programs/hugePage/common/eal_common_memzone.c Normal file
View File

@@ -0,0 +1,432 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include <stdint.h>
#include <inttypes.h>
#include <string.h>
#include <errno.h>
#include "rte_log.h"
#include "rte_memory.h"
#include "rte_memzone.h"
#include "rte_eal.h"
#include "rte_errno.h"
#include "rte_string_fns.h"
#include "rte_common.h"
#include "rte_eal_trace.h"
#include "malloc_heap.h"
#include "malloc_elem.h"
#include "eal_private.h"
#include "eal_memcfg.h"
static inline const struct rte_memzone *
memzone_lookup_thread_unsafe(const char *name)
{
struct rte_mem_config *mcfg;
struct rte_fbarray *arr;
const struct rte_memzone *mz;
int i = 0;
/* get pointer to global configuration */
mcfg = rte_eal_get_configuration()->mem_config;
arr = &mcfg->memzones;
/*
* the algorithm is not optimal (linear), but there are few
* zones and this function should be called at init only
*/
i = rte_fbarray_find_next_used(arr, 0);
while (i >= 0) {
mz = rte_fbarray_get(arr, i);
if (mz->addr != NULL &&
!strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE))
return mz;
i = rte_fbarray_find_next_used(arr, i + 1);
}
return NULL;
}
#define MEMZONE_KNOWN_FLAGS (RTE_MEMZONE_2MB \
| RTE_MEMZONE_1GB \
| RTE_MEMZONE_16MB \
| RTE_MEMZONE_16GB \
| RTE_MEMZONE_256KB \
| RTE_MEMZONE_256MB \
| RTE_MEMZONE_512MB \
| RTE_MEMZONE_4GB \
| RTE_MEMZONE_SIZE_HINT_ONLY \
| RTE_MEMZONE_IOVA_CONTIG \
)
static const struct rte_memzone *
memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
int socket_id, unsigned int flags, unsigned int align,
unsigned int bound)
{
struct rte_memzone *mz;
struct rte_mem_config *mcfg;
struct rte_fbarray *arr;
void *mz_addr;
size_t requested_len;
int mz_idx;
bool contig;
/* get pointer to global configuration */
mcfg = rte_eal_get_configuration()->mem_config;
arr = &mcfg->memzones;
/* no more room in config */
if (arr->count >= arr->len) {
RTE_LOG(ERR, EAL,
"%s(): Number of requested memzone segments exceeds RTE_MAX_MEMZONE\n",
__func__);
rte_errno = ENOSPC;
return NULL;
}
if (strlen(name) > sizeof(mz->name) - 1) {
RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n",
__func__, name);
rte_errno = ENAMETOOLONG;
return NULL;
}
/* zone already exist */
if ((memzone_lookup_thread_unsafe(name)) != NULL) {
RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n",
__func__, name);
rte_errno = EEXIST;
return NULL;
}
/* if alignment is not a power of two */
if (align && !rte_is_power_of_2(align)) {
RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__,
align);
rte_errno = EINVAL;
return NULL;
}
/* alignment less than cache size is not allowed */
if (align < RTE_CACHE_LINE_SIZE)
align = RTE_CACHE_LINE_SIZE;
/* align length on cache boundary. Check for overflow before doing so */
if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) {
rte_errno = EINVAL; /* requested size too big */
return NULL;
}
len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE);
/* save minimal requested length */
requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len);
/* check that boundary condition is valid */
if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) {
rte_errno = EINVAL;
return NULL;
}
if ((socket_id != SOCKET_ID_ANY) && socket_id < 0) {
rte_errno = EINVAL;
return NULL;
}
if ((flags & ~MEMZONE_KNOWN_FLAGS) != 0) {
rte_errno = EINVAL;
return NULL;
}
/* only set socket to SOCKET_ID_ANY if we aren't allocating for an
* external heap.
*/
if (!rte_eal_has_hugepages() && socket_id < RTE_MAX_NUMA_NODES)
socket_id = SOCKET_ID_ANY;
contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0;
/* malloc only cares about size flags, remove contig flag from flags */
flags &= ~RTE_MEMZONE_IOVA_CONTIG;
if (len == 0 && bound == 0) {
/* no size constraints were placed, so use malloc elem len */
requested_len = 0;
mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags,
align, contig);
} else {
if (len == 0)
requested_len = bound;
/* allocate memory on heap */
mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id,
flags, align, bound, contig);
}
if (mz_addr == NULL) {
rte_errno = ENOMEM;
return NULL;
}
struct malloc_elem *elem = malloc_elem_from_data(mz_addr);
/* fill the zone in config */
mz_idx = rte_fbarray_find_next_free(arr, 0);
if (mz_idx < 0) {
mz = NULL;
} else {
rte_fbarray_set_used(arr, mz_idx);
mz = rte_fbarray_get(arr, mz_idx);
}
if (mz == NULL) {
RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone\n", __func__);
malloc_heap_free(elem);
rte_errno = ENOSPC;
return NULL;
}
strlcpy(mz->name, name, sizeof(mz->name));
mz->iova = rte_malloc_virt2iova(mz_addr);
mz->addr = mz_addr;
mz->len = requested_len == 0 ?
elem->size - elem->pad - MALLOC_ELEM_OVERHEAD :
requested_len;
mz->hugepage_sz = elem->msl->page_sz;
mz->socket_id = elem->msl->socket_id;
mz->flags = 0;
return mz;
}
static const struct rte_memzone *
rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
unsigned int flags, unsigned int align, unsigned int bound)
{
struct rte_mem_config *mcfg;
const struct rte_memzone *mz = NULL;
/* get pointer to global configuration */
mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_lock(&mcfg->mlock);
mz = memzone_reserve_aligned_thread_unsafe(
name, len, socket_id, flags, align, bound);
rte_eal_trace_memzone_reserve(name, len, socket_id, flags, align,
bound, mz);
rte_rwlock_write_unlock(&mcfg->mlock);
return mz;
}
/*
* Return a pointer to a correctly filled memzone descriptor (with a
* specified alignment and boundary). If the allocation cannot be done,
* return NULL.
*/
const struct rte_memzone *
rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
unsigned flags, unsigned align, unsigned bound)
{
return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
align, bound);
}
/*
* Return a pointer to a correctly filled memzone descriptor (with a
* specified alignment). If the allocation cannot be done, return NULL.
*/
const struct rte_memzone *
rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
unsigned flags, unsigned align)
{
return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
align, 0);
}
/*
* Return a pointer to a correctly filled memzone descriptor. If the
* allocation cannot be done, return NULL.
*/
const struct rte_memzone *
rte_memzone_reserve(const char *name, size_t len, int socket_id,
unsigned flags)
{
return rte_memzone_reserve_thread_safe(name, len, socket_id,
flags, RTE_CACHE_LINE_SIZE, 0);
}
int
rte_memzone_free(const struct rte_memzone *mz)
{
char name[RTE_MEMZONE_NAMESIZE];
struct rte_mem_config *mcfg;
struct rte_fbarray *arr;
struct rte_memzone *found_mz;
int ret = 0;
void *addr = NULL;
unsigned idx;
if (mz == NULL)
return -EINVAL;
rte_strlcpy(name, mz->name, RTE_MEMZONE_NAMESIZE);
mcfg = rte_eal_get_configuration()->mem_config;
arr = &mcfg->memzones;
rte_rwlock_write_lock(&mcfg->mlock);
idx = rte_fbarray_find_idx(arr, mz);
found_mz = rte_fbarray_get(arr, idx);
if (found_mz == NULL) {
ret = -EINVAL;
} else if (found_mz->addr == NULL) {
RTE_LOG(ERR, EAL, "Memzone is not allocated\n");
ret = -EINVAL;
} else {
addr = found_mz->addr;
memset(found_mz, 0, sizeof(*found_mz));
rte_fbarray_set_free(arr, idx);
}
rte_rwlock_write_unlock(&mcfg->mlock);
rte_free(addr);
rte_eal_trace_memzone_free(name, addr, ret);
return ret;
}
/*
* Lookup for the memzone identified by the given name
*/
const struct rte_memzone *
rte_memzone_lookup(const char *name)
{
struct rte_mem_config *mcfg;
const struct rte_memzone *memzone = NULL;
mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_read_lock(&mcfg->mlock);
memzone = memzone_lookup_thread_unsafe(name);
rte_rwlock_read_unlock(&mcfg->mlock);
rte_eal_trace_memzone_lookup(name, memzone);
return memzone;
}
static void
dump_memzone(const struct rte_memzone *mz, void *arg)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
struct rte_memseg_list *msl = NULL;
void *cur_addr, *mz_end;
struct rte_memseg *ms;
int mz_idx, ms_idx;
size_t page_sz;
FILE *f = arg;
mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz);
fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, "
"socket_id:%"PRId32", flags:%"PRIx32"\n",
mz_idx,
mz->name,
mz->len,
mz->addr,
mz->socket_id,
mz->flags);
/* go through each page occupied by this memzone */
msl = rte_mem_virt2memseg_list(mz->addr);
if (!msl) {
RTE_LOG(DEBUG, EAL, "Skipping bad memzone\n");
return;
}
page_sz = (size_t)mz->hugepage_sz;
cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz);
mz_end = RTE_PTR_ADD(cur_addr, mz->len);
fprintf(f, "physical segments used:\n");
ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz;
ms = rte_fbarray_get(&msl->memseg_arr, ms_idx);
do {
fprintf(f, " addr: %p iova: 0x%" PRIx64 " "
"len: 0x%zx "
"pagesz: 0x%zx\n",
cur_addr, ms->iova, ms->len, page_sz);
/* advance VA to next page */
cur_addr = RTE_PTR_ADD(cur_addr, page_sz);
/* memzones occupy contiguous segments */
++ms;
} while (cur_addr < mz_end);
}
/* Dump all reserved memory zones on console */
void
rte_memzone_dump(FILE *f)
{
rte_memzone_walk(dump_memzone, f);
}
/*
* Init the memzone subsystem
*/
int
rte_eal_memzone_init(void)
{
struct rte_mem_config *mcfg;
int ret = 0;
/* get pointer to global configuration */
mcfg = rte_eal_get_configuration()->mem_config;
rte_rwlock_write_lock(&mcfg->mlock);
if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
rte_fbarray_init(&mcfg->memzones, "memzone",
RTE_MAX_MEMZONE, sizeof(struct rte_memzone))) {
RTE_LOG(ERR, EAL, "Cannot allocate memzone list\n");
ret = -1;
} else if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
rte_fbarray_attach(&mcfg->memzones)) {
RTE_LOG(ERR, EAL, "Cannot attach to memzone list\n");
ret = -1;
}
rte_rwlock_write_unlock(&mcfg->mlock);
return ret;
}
/* Walk all reserved memory zones */
void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *),
void *arg)
{
struct rte_mem_config *mcfg;
struct rte_fbarray *arr;
int i;
mcfg = rte_eal_get_configuration()->mem_config;
arr = &mcfg->memzones;
rte_rwlock_read_lock(&mcfg->mlock);
i = rte_fbarray_find_next_used(arr, 0);
while (i >= 0) {
struct rte_memzone *mz = rte_fbarray_get(arr, i);
(*func)(mz, arg);
i = rte_fbarray_find_next_used(arr, i + 1);
}
rte_rwlock_read_unlock(&mcfg->mlock);
}

2246
Programs/hugePage/common/eal_common_options.c Normal file
View File

File diff suppressed because it is too large Load Diff

1291
Programs/hugePage/common/eal_common_proc.c Normal file
View File

File diff suppressed because it is too large Load Diff

100
Programs/hugePage/common/eal_common_string_fns.c Normal file
View File

@@ -0,0 +1,100 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include "rte_string_fns.h"
#include "rte_errno.h"
/* split string into tokens */
int
rte_strsplit(char *string, int stringlen,
char **tokens, int maxtokens, char delim)
{
int i, tok = 0;
int tokstart = 1; /* first token is right at start of string */
if (string == NULL || tokens == NULL)
goto einval_error;
for (i = 0; i < stringlen; i++) {
if (string[i] == '\0' || tok >= maxtokens)
break;
if (tokstart) {
tokstart = 0;
tokens[tok++] = &string[i];
}
if (string[i] == delim) {
string[i] = '\0';
tokstart = 1;
}
}
return tok;
einval_error:
errno = EINVAL;
return -1;
}
/* Copy src string into dst.
*
* Return negative value and NUL-terminate if dst is too short,
* Otherwise return number of bytes copied.
*/
ssize_t
rte_strscpy(char *dst, const char *src, size_t dsize)
{
size_t nleft = dsize;
size_t res = 0;
/* Copy as many bytes as will fit. */
while (nleft != 0) {
dst[res] = src[res];
if (src[res] == '\0')
return res;
res++;
nleft--;
}
/* Not enough room in dst, set NUL and return error. */
if (res != 0)
dst[res - 1] = '\0';
rte_errno = E2BIG;
return -rte_errno;
}
uint64_t
rte_str_to_size(const char *str)
{
char *endptr;
unsigned long long size;
while (isspace((int)*str))
str++;
if (*str == '-')
return 0;
errno = 0;
size = strtoull(str, &endptr, 0);
if (errno)
return 0;
if (*endptr == ' ')
endptr++; /* allow 1 space gap */
switch (*endptr) {
case 'G': case 'g':
size *= 1024; /* fall-through */
case 'M': case 'm':
size *= 1024; /* fall-through */
case 'K': case 'k':
size *= 1024; /* fall-through */
default:
break;
}
return size;
}

160
Programs/hugePage/common/eal_common_tailqs.c Normal file
View File

@@ -0,0 +1,160 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <sys/queue.h>
#include <stdio.h>
#include <string.h>
#include "rte_eal.h"
#include "rte_eal_memconfig.h"
#include "rte_log.h"
#include "rte_string_fns.h"
#include "eal_private.h"
#include "eal_memcfg.h"
TAILQ_HEAD(rte_tailq_elem_head, rte_tailq_elem);
/* local tailq list */
static struct rte_tailq_elem_head rte_tailq_elem_head =
TAILQ_HEAD_INITIALIZER(rte_tailq_elem_head);
/* number of tailqs registered, -1 before call to rte_eal_tailqs_init */
static int rte_tailqs_count = -1;
struct rte_tailq_head *
rte_eal_tailq_lookup(const char *name)
{
unsigned i;
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
if (name == NULL)
return NULL;
for (i = 0; i < RTE_MAX_TAILQ; i++) {
if (!strncmp(name, mcfg->tailq_head[i].name,
RTE_TAILQ_NAMESIZE-1))
return &mcfg->tailq_head[i];
}
return NULL;
}
void
rte_dump_tailq(FILE *f)
{
struct rte_mem_config *mcfg;
unsigned i = 0;
mcfg = rte_eal_get_configuration()->mem_config;
rte_mcfg_tailq_read_lock();
for (i = 0; i < RTE_MAX_TAILQ; i++) {
const struct rte_tailq_head *tailq = &mcfg->tailq_head[i];
const struct rte_tailq_entry_head *head = &tailq->tailq_head;
fprintf(f, "Tailq %u: qname:<%s>, tqh_first:%p, tqh_last:%p\n",
i, tailq->name, head->tqh_first, head->tqh_last);
}
rte_mcfg_tailq_read_unlock();
}
static struct rte_tailq_head *
rte_eal_tailq_create(const char *name)
{
struct rte_tailq_head *head = NULL;
if (!rte_eal_tailq_lookup(name) &&
(rte_tailqs_count + 1 < RTE_MAX_TAILQ)) {
struct rte_mem_config *mcfg;
mcfg = rte_eal_get_configuration()->mem_config;
head = &mcfg->tailq_head[rte_tailqs_count];
strlcpy(head->name, name, sizeof(head->name) - 1);
TAILQ_INIT(&head->tailq_head);
rte_tailqs_count++;
}
return head;
}
/* local register, used to store "early" tailqs before rte_eal_init() and to
* ensure secondary process only registers tailqs once. */
static int
rte_eal_tailq_local_register(struct rte_tailq_elem *t)
{
struct rte_tailq_elem *temp;
TAILQ_FOREACH(temp, &rte_tailq_elem_head, next) {
if (!strncmp(t->name, temp->name, sizeof(temp->name)))
return -1;
}
TAILQ_INSERT_TAIL(&rte_tailq_elem_head, t, next);
return 0;
}
static void
rte_eal_tailq_update(struct rte_tailq_elem *t)
{
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
/* primary process is the only one that creates */
t->head = rte_eal_tailq_create(t->name);
} else {
t->head = rte_eal_tailq_lookup(t->name);
}
}
int
rte_eal_tailq_register(struct rte_tailq_elem *t)
{
if (rte_eal_tailq_local_register(t) < 0) {
RTE_LOG(ERR, EAL,
"%s tailq is already registered\n", t->name);
goto error;
}
/* if a register happens after rte_eal_tailqs_init(), then we can update
* tailq head */
if (rte_tailqs_count >= 0) {
rte_eal_tailq_update(t);
if (t->head == NULL) {
RTE_LOG(ERR, EAL,
"Cannot initialize tailq: %s\n", t->name);
TAILQ_REMOVE(&rte_tailq_elem_head, t, next);
goto error;
}
}
return 0;
error:
t->head = NULL;
return -1;
}
int
rte_eal_tailqs_init(void)
{
struct rte_tailq_elem *t;
rte_tailqs_count = 0;
TAILQ_FOREACH(t, &rte_tailq_elem_head, next) {
/* second part of register job for "early" tailqs, see
* rte_eal_tailq_register and EAL_REGISTER_TAILQ */
rte_eal_tailq_update(t);
if (t->head == NULL) {
RTE_LOG(ERR, EAL,
"Cannot initialize tailq: %s\n", t->name);
/* TAILQ_REMOVE not needed, error is already fatal */
goto fail;
}
}
return 0;
fail:
rte_dump_tailq(stderr);
return -1;
}

423
Programs/hugePage/common/eal_common_thread.c Normal file
View File

@@ -0,0 +1,423 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <sched.h>
#include <assert.h>
#include <string.h>
#include <inttypes.h>
#include "rte_eal_trace.h"
#include "rte_errno.h"
#include "rte_lcore.h"
#include "rte_log.h"
#include "rte_memory.h"
#include "rte_trace_point.h"
#include "eal_internal_cfg.h"
#include "eal_private.h"
#include "eal_thread.h"
#include "eal_trace.h"
RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
RTE_DEFINE_PER_LCORE(int, _thread_id) = -1;
static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) =
(unsigned int)SOCKET_ID_ANY;
static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset);
unsigned rte_socket_id(void)
{
return RTE_PER_LCORE(_socket_id);
}
static int
eal_cpuset_socket_id(rte_cpuset_t *cpusetp)
{
unsigned cpu = 0;
int socket_id = SOCKET_ID_ANY;
int sid;
if (cpusetp == NULL)
return SOCKET_ID_ANY;
do {
if (!CPU_ISSET(cpu, cpusetp))
continue;
if (socket_id == SOCKET_ID_ANY)
socket_id = eal_cpu_socket_id(cpu);
sid = eal_cpu_socket_id(cpu);
if (socket_id != sid) {
socket_id = SOCKET_ID_ANY;
break;
}
} while (++cpu < CPU_SETSIZE);
return socket_id;
}
static void
thread_update_affinity(rte_cpuset_t *cpusetp)
{
unsigned int lcore_id = rte_lcore_id();
/* store socket_id in TLS for quick access */
RTE_PER_LCORE(_socket_id) =
eal_cpuset_socket_id(cpusetp);
/* store cpuset in TLS for quick access */
memmove(&RTE_PER_LCORE(_cpuset), cpusetp,
sizeof(rte_cpuset_t));
if (lcore_id != (unsigned)LCORE_ID_ANY) {
/* EAL thread will update lcore_config */
lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id);
memmove(&lcore_config[lcore_id].cpuset, cpusetp,
sizeof(rte_cpuset_t));
}
}
int
rte_thread_set_affinity(rte_cpuset_t *cpusetp)
{
if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
cpusetp) != 0) {
RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n");
return -1;
}
thread_update_affinity(cpusetp);
return 0;
}
void
rte_thread_get_affinity(rte_cpuset_t *cpusetp)
{
assert(cpusetp);
memmove(cpusetp, &RTE_PER_LCORE(_cpuset),
sizeof(rte_cpuset_t));
}
int
eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size)
{
unsigned cpu;
int ret;
unsigned int out = 0;
for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
if (!CPU_ISSET(cpu, cpuset))
continue;
ret = snprintf(str + out,
size - out, "%u,", cpu);
if (ret < 0 || (unsigned)ret >= size - out) {
/* string will be truncated */
ret = -1;
goto exit;
}
out += ret;
}
ret = 0;
exit:
/* remove the last separator */
if (out > 0)
str[out - 1] = '\0';
return ret;
}
int
eal_thread_dump_current_affinity(char *str, unsigned int size)
{
rte_cpuset_t cpuset;
rte_thread_get_affinity(&cpuset);
return eal_thread_dump_affinity(&cpuset, str, size);
}
void
__rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset)
{
/* set the lcore ID in per-lcore memory area */
RTE_PER_LCORE(_lcore_id) = lcore_id;
/* acquire system unique id */
rte_gettid();
thread_update_affinity(cpuset);
__rte_trace_mem_per_thread_alloc();
}
void
__rte_thread_uninit(void)
{
trace_mem_per_thread_free();
RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY;
}
/* main loop of threads */
__rte_noreturn void *
eal_thread_loop(void *arg)
{
unsigned int lcore_id = (uintptr_t)arg;
char cpuset[RTE_CPU_AFFINITY_STR_LEN];
int ret;
__rte_thread_init(lcore_id, &lcore_config[lcore_id].cpuset);
ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%" PRIxPTR ";cpuset=[%s%s])\n",
lcore_id, (uintptr_t)pthread_self(), cpuset,
ret == 0 ? "" : "...");
rte_eal_trace_thread_lcore_ready(lcore_id, cpuset);
/* read on our pipe to get commands */
while (1) {
lcore_function_t *f;
void *fct_arg;
eal_thread_wait_command();
/* Set the state to 'RUNNING'. Use release order
* since 'state' variable is used as the guard variable.
*/
__atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
__ATOMIC_RELEASE);
eal_thread_ack_command();
/* Load 'f' with acquire order to ensure that
* the memory operations from the main thread
* are accessed only after update to 'f' is visible.
* Wait till the update to 'f' is visible to the worker.
*/
while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
__ATOMIC_ACQUIRE)) == NULL)
rte_pause();
/* call the function and store the return value */
fct_arg = lcore_config[lcore_id].arg;
ret = f(fct_arg);
lcore_config[lcore_id].ret = ret;
lcore_config[lcore_id].f = NULL;
lcore_config[lcore_id].arg = NULL;
/* Store the state with release order to ensure that
* the memory operations from the worker thread
* are completed before the state is updated.
* Use 'state' as the guard variable.
*/
__atomic_store_n(&lcore_config[lcore_id].state, WAIT,
__ATOMIC_RELEASE);
}
/* never reached */
/* pthread_exit(NULL); */
/* return NULL; */
}
enum __rte_ctrl_thread_status {
CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */
CTRL_THREAD_RUNNING, /* Control thread is running successfully */
CTRL_THREAD_ERROR /* Control thread encountered an error */
};
struct rte_thread_ctrl_params {
void *(*start_routine)(void *);
void *arg;
int ret;
/* Control thread status.
* If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
*/
enum __rte_ctrl_thread_status ctrl_thread_status;
};
static void *ctrl_thread_init(void *arg)
{
struct internal_config *internal_conf =
eal_get_internal_configuration();
rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
struct rte_thread_ctrl_params *params = arg;
void *(*start_routine)(void *) = params->start_routine;
void *routine_arg = params->arg;
__rte_thread_init(rte_lcore_id(), cpuset);
params->ret = pthread_setaffinity_np(pthread_self(), sizeof(*cpuset),
cpuset);
if (params->ret != 0) {
__atomic_store_n(&params->ctrl_thread_status,
CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
return NULL;
}
__atomic_store_n(&params->ctrl_thread_status,
CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
return start_routine(routine_arg);
}
int
rte_ctrl_thread_create(pthread_t *thread, const char *name,
const pthread_attr_t *attr,
void *(*start_routine)(void *), void *arg)
{
struct rte_thread_ctrl_params *params;
enum __rte_ctrl_thread_status ctrl_thread_status;
int ret;
params = malloc(sizeof(*params));
if (!params)
return -ENOMEM;
params->start_routine = start_routine;
params->arg = arg;
params->ret = 0;
params->ctrl_thread_status = CTRL_THREAD_LAUNCHING;
ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
if (ret != 0) {
free(params);
return -ret;
}
if (name != NULL) {
ret = rte_thread_setname(*thread, name);
if (ret < 0)
RTE_LOG(DEBUG, EAL,
"Cannot set name for ctrl thread\n");
}
/* Wait for the control thread to initialize successfully */
while ((ctrl_thread_status =
__atomic_load_n(&params->ctrl_thread_status,
__ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
/* Yield the CPU. Using sched_yield call requires maintaining
* another implementation for Windows as sched_yield is not
* supported on Windows.
*/
rte_delay_us_sleep(1);
}
/* Check if the control thread encountered an error */
if (ctrl_thread_status == CTRL_THREAD_ERROR) {
/* ctrl thread is exiting */
pthread_join(*thread, NULL);
}
ret = params->ret;
free(params);
return -ret;
}
int
rte_thread_register(void)
{
unsigned int lcore_id;
rte_cpuset_t cpuset;
/* EAL init flushes all lcores, we can't register before. */
if (eal_get_internal_configuration()->init_complete != 1) {
RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__);
rte_errno = EINVAL;
return -1;
}
if (!rte_mp_disable()) {
RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n");
rte_errno = EINVAL;
return -1;
}
if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset),
&cpuset) != 0)
CPU_ZERO(&cpuset);
lcore_id = eal_lcore_non_eal_allocate();
if (lcore_id >= RTE_MAX_LCORE)
lcore_id = LCORE_ID_ANY;
__rte_thread_init(lcore_id, &cpuset);
if (lcore_id == LCORE_ID_ANY) {
rte_errno = ENOMEM;
return -1;
}
RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n",
lcore_id);
return 0;
}
void
rte_thread_unregister(void)
{
unsigned int lcore_id = rte_lcore_id();
if (lcore_id != LCORE_ID_ANY)
eal_lcore_non_eal_release(lcore_id);
__rte_thread_uninit();
if (lcore_id != LCORE_ID_ANY)
RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n",
lcore_id);
}
int
rte_thread_attr_init(rte_thread_attr_t *attr)
{
if (attr == NULL)
return EINVAL;
CPU_ZERO(&attr->cpuset);
attr->priority = RTE_THREAD_PRIORITY_NORMAL;
return 0;
}
int
rte_thread_attr_set_priority(rte_thread_attr_t *thread_attr,
enum rte_thread_priority priority)
{
if (thread_attr == NULL)
return EINVAL;
thread_attr->priority = priority;
return 0;
}
int
rte_thread_attr_set_affinity(rte_thread_attr_t *thread_attr,
rte_cpuset_t *cpuset)
{
if (thread_attr == NULL)
return EINVAL;
if (cpuset == NULL)
return EINVAL;
thread_attr->cpuset = *cpuset;
return 0;
}
int
rte_thread_attr_get_affinity(rte_thread_attr_t *thread_attr,
rte_cpuset_t *cpuset)
{
if (thread_attr == NULL)
return EINVAL;
if (cpuset == NULL)
return EINVAL;
*cpuset = thread_attr->cpuset;
return 0;
}

88
Programs/hugePage/common/eal_common_timer.c Normal file
View File

@@ -0,0 +1,88 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include <inttypes.h>
#include "rte_common.h"
#include "rte_log.h"
#include "rte_cycles.h"
#include "rte_pause.h"
#include "rte_eal.h"
#include "eal_private.h"
#include "eal_memcfg.h"
/* The frequency of the RDTSC timer resolution */
static uint64_t eal_tsc_resolution_hz;
/* Pointer to user delay function */
void (*rte_delay_us)(unsigned int) = NULL;
void
rte_delay_us_block(unsigned int us)
{
const uint64_t start = rte_get_timer_cycles();
const uint64_t ticks = (uint64_t)us * rte_get_timer_hz() / 1E6;
while ((rte_get_timer_cycles() - start) < ticks)
rte_pause();
}
uint64_t
rte_get_tsc_hz(void)
{
return eal_tsc_resolution_hz;
}
static uint64_t
estimate_tsc_freq(void)
{
#define CYC_PER_10MHZ 1E7
RTE_LOG(WARNING, EAL, "WARNING: TSC frequency estimated roughly"
" - clock timings may be less accurate.\n");
/* assume that the rte_delay_us_sleep() will sleep for 1 second */
uint64_t start = rte_rdtsc();
rte_delay_us_sleep(US_PER_S);
/* Round up to 10Mhz. 1E7 ~ 10Mhz */
return RTE_ALIGN_MUL_NEAR(rte_rdtsc() - start, CYC_PER_10MHZ);
}
void
set_tsc_freq(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
uint64_t freq;
if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
/*
* Just use the primary process calculated TSC rate in any
* secondary process. It avoids any unnecessary overhead on
* systems where arch-specific frequency detection is not
* available.
*/
eal_tsc_resolution_hz = mcfg->tsc_hz;
return;
}
freq = get_tsc_freq_arch();
if (!freq)
freq = get_tsc_freq();
if (!freq)
freq = estimate_tsc_freq();
RTE_LOG(DEBUG, EAL, "TSC frequency is ~%" PRIu64 " KHz\n", freq / 1000);
eal_tsc_resolution_hz = freq;
mcfg->tsc_hz = freq;
}
void rte_delay_us_callback_register(void (*userfunc)(unsigned int))
{
rte_delay_us = userfunc;
}
RTE_INIT(rte_timer_init)
{
/* set rte_delay_us_block as a delay function */
rte_delay_us_callback_register(rte_delay_us_block);
}

523
Programs/hugePage/common/eal_common_trace.c Normal file
View File

@@ -0,0 +1,523 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2020 Marvell International Ltd.
*/
#include <stdlib.h>
#include <fnmatch.h>
#include <sys/queue.h>
#include <regex.h>
#include "rte_common.h"
#include "rte_errno.h"
#include "rte_lcore.h"
#include "rte_per_lcore.h"
#include "rte_string_fns.h"
#include "eal_trace.h"
RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz);
RTE_DEFINE_PER_LCORE(void *, trace_mem);
static RTE_DEFINE_PER_LCORE(char *, ctf_field);
static struct trace_point_head tp_list = STAILQ_HEAD_INITIALIZER(tp_list);
static struct trace trace = { .args = STAILQ_HEAD_INITIALIZER(trace.args), };
struct trace *
trace_obj_get(void)
{
return &trace;
}
struct trace_point_head *
trace_list_head_get(void)
{
return &tp_list;
}
int
eal_trace_init(void)
{
struct trace_arg *arg;
/* Trace memory should start with 8B aligned for natural alignment */
RTE_BUILD_BUG_ON((offsetof(struct __rte_trace_header, mem) % 8) != 0);
/* One of the trace point registration failed */
if (trace.register_errno) {
rte_errno = trace.register_errno;
goto fail;
}
rte_spinlock_init(&trace.lock);
/* Is duplicate trace name registered */
if (trace_has_duplicate_entry())
goto fail;
/* Generate UUID ver 4 with total size of events and number of
* events
*/
trace_uuid_generate();
/* Apply buffer size configuration for trace output */
trace_bufsz_args_apply();
/* Generate CTF TDSL metadata */
if (trace_metadata_create() < 0)
goto fail;
/* Save current epoch timestamp for future use */
if (trace_epoch_time_save() < 0)
goto free_meta;
/* Apply global configurations */
STAILQ_FOREACH(arg, &trace.args, next)
trace_args_apply(arg->val);
rte_trace_mode_set(trace.mode);
return 0;
free_meta:
trace_metadata_destroy();
fail:
trace_err("failed to initialize trace [%s]", rte_strerror(rte_errno));
return -rte_errno;
}
void
eal_trace_fini(void)
{
trace_mem_free();
trace_metadata_destroy();
eal_trace_args_free();
}
bool
rte_trace_is_enabled(void)
{
return __atomic_load_n(&trace.status, __ATOMIC_ACQUIRE) != 0;
}
static void
trace_mode_set(rte_trace_point_t *t, enum rte_trace_mode mode)
{
if (mode == RTE_TRACE_MODE_OVERWRITE)
__atomic_and_fetch(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD,
__ATOMIC_RELEASE);
else
__atomic_or_fetch(t, __RTE_TRACE_FIELD_ENABLE_DISCARD,
__ATOMIC_RELEASE);
}
void
rte_trace_mode_set(enum rte_trace_mode mode)
{
struct trace_point *tp;
STAILQ_FOREACH(tp, &tp_list, next)
trace_mode_set(tp->handle, mode);
trace.mode = mode;
}
enum
rte_trace_mode rte_trace_mode_get(void)
{
return trace.mode;
}
static bool
trace_point_is_invalid(rte_trace_point_t *t)
{
return (t == NULL) || (trace_id_get(t) >= trace.nb_trace_points);
}
bool
rte_trace_point_is_enabled(rte_trace_point_t *t)
{
uint64_t val;
if (trace_point_is_invalid(t))
return false;
val = __atomic_load_n(t, __ATOMIC_ACQUIRE);
return (val & __RTE_TRACE_FIELD_ENABLE_MASK) != 0;
}
int
rte_trace_point_enable(rte_trace_point_t *t)
{
uint64_t prev;
if (trace_point_is_invalid(t))
return -ERANGE;
prev = __atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE);
if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0)
__atomic_add_fetch(&trace.status, 1, __ATOMIC_RELEASE);
return 0;
}
int
rte_trace_point_disable(rte_trace_point_t *t)
{
uint64_t prev;
if (trace_point_is_invalid(t))
return -ERANGE;
prev = __atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE);
if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0)
__atomic_sub_fetch(&trace.status, 1, __ATOMIC_RELEASE);
return 0;
}
int
rte_trace_pattern(const char *pattern, bool enable)
{
struct trace_point *tp;
int rc = 0, found = 0;
STAILQ_FOREACH(tp, &tp_list, next) {
if (fnmatch(pattern, tp->name, 0) != 0)
continue;
if (enable)
rc = rte_trace_point_enable(tp->handle);
else
rc = rte_trace_point_disable(tp->handle);
if (rc < 0) {
found = 0;
break;
}
found = 1;
}
return rc | found;
}
int
rte_trace_regexp(const char *regex, bool enable)
{
struct trace_point *tp;
int rc = 0, found = 0;
regex_t r;
if (regcomp(&r, regex, 0) != 0)
return -EINVAL;
STAILQ_FOREACH(tp, &tp_list, next) {
if (regexec(&r, tp->name, 0, NULL, 0) != 0)
continue;
if (enable)
rc = rte_trace_point_enable(tp->handle);
else
rc = rte_trace_point_disable(tp->handle);
if (rc < 0) {
found = 0;
break;
}
found = 1;
}
regfree(&r);
return rc | found;
}
rte_trace_point_t *
rte_trace_point_lookup(const char *name)
{
struct trace_point *tp;
if (name == NULL)
return NULL;
STAILQ_FOREACH(tp, &tp_list, next)
if (strcmp(tp->name, name) == 0)
return tp->handle;
return NULL;
}
static void
trace_point_dump(FILE *f, struct trace_point *tp)
{
rte_trace_point_t *handle = tp->handle;
fprintf(f, "\tid %d, %s, size is %d, %s\n",
trace_id_get(handle), tp->name,
(uint16_t)(*handle & __RTE_TRACE_FIELD_SIZE_MASK),
rte_trace_point_is_enabled(handle) ? "enabled" : "disabled");
}
static void
trace_lcore_mem_dump(FILE *f)
{
struct trace *trace = trace_obj_get();
struct __rte_trace_header *header;
uint32_t count;
rte_spinlock_lock(&trace->lock);
if (trace->nb_trace_mem_list == 0)
goto out;
fprintf(f, "nb_trace_mem_list = %d\n", trace->nb_trace_mem_list);
fprintf(f, "\nTrace mem info\n--------------\n");
for (count = 0; count < trace->nb_trace_mem_list; count++) {
header = trace->lcore_meta[count].mem;
fprintf(f, "\tid %d, mem=%p, area=%s, lcore_id=%d, name=%s\n",
count, header,
trace_area_to_string(trace->lcore_meta[count].area),
header->stream_header.lcore_id,
header->stream_header.thread_name);
}
out:
rte_spinlock_unlock(&trace->lock);
}
void
rte_trace_dump(FILE *f)
{
struct trace_point_head *tp_list = trace_list_head_get();
struct trace *trace = trace_obj_get();
struct trace_point *tp;
fprintf(f, "\nGlobal info\n-----------\n");
fprintf(f, "status = %s\n",
rte_trace_is_enabled() ? "enabled" : "disabled");
fprintf(f, "mode = %s\n",
trace_mode_to_string(rte_trace_mode_get()));
fprintf(f, "dir = %s\n", trace->dir);
fprintf(f, "buffer len = %d\n", trace->buff_len);
fprintf(f, "number of trace points = %d\n", trace->nb_trace_points);
trace_lcore_mem_dump(f);
fprintf(f, "\nTrace point info\n----------------\n");
STAILQ_FOREACH(tp, tp_list, next)
trace_point_dump(f, tp);
}
void
__rte_trace_mem_per_thread_alloc(void)
{
struct trace *trace = trace_obj_get();
struct __rte_trace_header *header;
uint32_t count;
if (!rte_trace_is_enabled())
return;
if (RTE_PER_LCORE(trace_mem))
return;
rte_spinlock_lock(&trace->lock);
count = trace->nb_trace_mem_list;
/* Allocate room for storing the thread trace mem meta */
trace->lcore_meta = realloc(trace->lcore_meta,
sizeof(trace->lcore_meta[0]) * (count + 1));
/* Provide dummy space for fast path to consume */
if (trace->lcore_meta == NULL) {
trace_crit("trace mem meta memory realloc failed");
header = NULL;
goto fail;
}
/* First attempt from huge page */
header = eal_malloc_no_trace(NULL, trace_mem_sz(trace->buff_len), 8);
if (header) {
trace->lcore_meta[count].area = TRACE_AREA_HUGEPAGE;
goto found;
}
/* Second attempt from heap */
header = malloc(trace_mem_sz(trace->buff_len));
if (header == NULL) {
trace_crit("trace mem malloc attempt failed");
header = NULL;
goto fail;
}
/* Second attempt from heap is success */
trace->lcore_meta[count].area = TRACE_AREA_HEAP;
/* Initialize the trace header */
found:
header->offset = 0;
header->len = trace->buff_len;
header->stream_header.magic = TRACE_CTF_MAGIC;
rte_uuid_copy(header->stream_header.uuid, trace->uuid);
header->stream_header.lcore_id = rte_lcore_id();
/* Store the thread name */
char *name = header->stream_header.thread_name;
memset(name, 0, __RTE_TRACE_EMIT_STRING_LEN_MAX);
rte_thread_getname(pthread_self(), name,
__RTE_TRACE_EMIT_STRING_LEN_MAX);
trace->lcore_meta[count].mem = header;
trace->nb_trace_mem_list++;
fail:
RTE_PER_LCORE(trace_mem) = header;
rte_spinlock_unlock(&trace->lock);
}
static void
trace_mem_per_thread_free_unlocked(struct thread_mem_meta *meta)
{
if (meta->area == TRACE_AREA_HUGEPAGE)
eal_free_no_trace(meta->mem);
else if (meta->area == TRACE_AREA_HEAP)
free(meta->mem);
}
void
trace_mem_per_thread_free(void)
{
struct trace *trace = trace_obj_get();
struct __rte_trace_header *header;
uint32_t count;
header = RTE_PER_LCORE(trace_mem);
if (header == NULL)
return;
rte_spinlock_lock(&trace->lock);
for (count = 0; count < trace->nb_trace_mem_list; count++) {
if (trace->lcore_meta[count].mem == header)
break;
}
if (count != trace->nb_trace_mem_list) {
struct thread_mem_meta *meta = &trace->lcore_meta[count];
trace_mem_per_thread_free_unlocked(meta);
if (count != trace->nb_trace_mem_list - 1) {
memmove(meta, meta + 1,
sizeof(*meta) *
(trace->nb_trace_mem_list - count - 1));
}
trace->nb_trace_mem_list--;
}
rte_spinlock_unlock(&trace->lock);
}
void
trace_mem_free(void)
{
struct trace *trace = trace_obj_get();
uint32_t count;
rte_spinlock_lock(&trace->lock);
for (count = 0; count < trace->nb_trace_mem_list; count++) {
trace_mem_per_thread_free_unlocked(&trace->lcore_meta[count]);
}
trace->nb_trace_mem_list = 0;
rte_spinlock_unlock(&trace->lock);
}
void
__rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype)
{
char *field;
char *fixup;
int rc;
fixup = trace_metadata_fixup_field(in);
if (fixup != NULL)
in = fixup;
rc = asprintf(&field, "%s %s %s;\n",
RTE_PER_LCORE(ctf_field) != NULL ?
RTE_PER_LCORE(ctf_field) : "",
datatype, in);
free(RTE_PER_LCORE(ctf_field));
free(fixup);
if (rc == -1) {
RTE_PER_LCORE(trace_point_sz) = 0;
RTE_PER_LCORE(ctf_field) = NULL;
trace_crit("could not allocate CTF field");
return;
}
RTE_PER_LCORE(trace_point_sz) += sz;
RTE_PER_LCORE(ctf_field) = field;
}
int
__rte_trace_point_register(rte_trace_point_t *handle, const char *name,
void (*register_fn)(void))
{
struct trace_point *tp;
uint16_t sz;
/* Sanity checks of arguments */
if (name == NULL || register_fn == NULL || handle == NULL) {
trace_err("invalid arguments");
rte_errno = EINVAL;
goto fail;
}
/* Check the size of the trace point object */
RTE_PER_LCORE(trace_point_sz) = 0;
register_fn();
if (RTE_PER_LCORE(trace_point_sz) == 0) {
trace_err("missing rte_trace_emit_header() in register fn");
rte_errno = EBADF;
goto fail;
}
/* Is size overflowed */
if (RTE_PER_LCORE(trace_point_sz) > UINT16_MAX) {
trace_err("trace point size overflowed");
rte_errno = ENOSPC;
goto fail;
}
/* Are we running out of space to store trace points? */
if (trace.nb_trace_points > UINT16_MAX) {
trace_err("trace point exceeds the max count");
rte_errno = ENOSPC;
goto fail;
}
/* Get the size of the trace point */
sz = RTE_PER_LCORE(trace_point_sz);
tp = calloc(1, sizeof(struct trace_point));
if (tp == NULL) {
trace_err("fail to allocate trace point memory");
rte_errno = ENOMEM;
goto fail;
}
/* Initialize the trace point */
tp->name = name;
/* Copy the accumulated fields description and clear it for the next
* trace point.
*/
tp->ctf_field = RTE_PER_LCORE(ctf_field);
RTE_PER_LCORE(ctf_field) = NULL;
/* Form the trace handle */
*handle = sz;
*handle |= trace.nb_trace_points << __RTE_TRACE_FIELD_ID_SHIFT;
trace_mode_set(handle, trace.mode);
trace.nb_trace_points++;
tp->handle = handle;
/* Add the trace point at tail */
STAILQ_INSERT_TAIL(&tp_list, tp, next);
__atomic_thread_fence(__ATOMIC_RELEASE);
/* All Good !!! */
return 0;
fail:
if (trace.register_errno == 0)
trace.register_errno = rte_errno;
return -rte_errno;
}

412
Programs/hugePage/common/eal_common_trace_ctf.c Normal file
View File

@@ -0,0 +1,412 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2020 Marvell International Ltd.
*/
#include <inttypes.h>
#include <stdlib.h>
#include <time.h>
#include "rte_byteorder.h"
#include "rte_common.h"
#include "rte_time.h"
#include "rte_trace.h"
#include "rte_version.h"
#include "eal_trace.h"
__rte_format_printf(2, 0)
static int
metadata_printf(char **str, const char *fmt, ...)
{
va_list ap;
int rc;
*str = NULL;
va_start(ap, fmt);
rc = vasprintf(str, fmt, ap);
va_end(ap);
return rc;
}
static int
meta_copy(char **meta, int *offset, char *str, int rc)
{
int count = *offset;
char *ptr = *meta;
if (rc < 0)
return rc;
ptr = realloc(ptr, count + rc + 1);
if (ptr == NULL)
goto free_str;
memcpy(RTE_PTR_ADD(ptr, count), str, rc);
ptr[count + rc] = '\0';
count += rc;
free(str);
*meta = ptr;
*offset = count;
return rc;
free_str:
free(str);
return -ENOMEM;
}
static int
meta_data_type_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"/* CTF 1.8 */\n"
"typealias integer {size = 8; base = x;}:= uint8_t;\n"
"typealias integer {size = 16; base = x;} := uint16_t;\n"
"typealias integer {size = 32; base = x;} := uint32_t;\n"
"typealias integer {size = 64; base = x;} := uint64_t;\n"
"typealias integer {size = 8; signed = true;} := int8_t;\n"
"typealias integer {size = 16; signed = true;} := int16_t;\n"
"typealias integer {size = 32; signed = true;} := int32_t;\n"
"typealias integer {size = 64; signed = true;} := int64_t;\n"
#ifdef RTE_ARCH_64
"typealias integer {size = 64; base = x;} := uintptr_t;\n"
#else
"typealias integer {size = 32; base = x;} := uintptr_t;\n"
#endif
#ifdef RTE_ARCH_64
"typealias integer {size = 64; base = x;} := long;\n"
#else
"typealias integer {size = 32; base = x;} := long;\n"
#endif
"typealias integer {size = 8; signed = false; encoding = ASCII; } := string_bounded_t;\n\n"
#ifdef RTE_ARCH_64
"typealias integer {size = 64; base = x;} := size_t;\n"
#else
"typealias integer {size = 32; base = x;} := size_t;\n"
#endif
"typealias floating_point {\n"
" exp_dig = 8;\n"
" mant_dig = 24;\n"
"} := float;\n\n"
"typealias floating_point {\n"
" exp_dig = 11;\n"
" mant_dig = 53;\n"
"} := double;\n\n");
return meta_copy(meta, offset, str, rc);
}
static int
is_be(void)
{
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
return 1;
#else
return 0;
#endif
}
static int
meta_header_emit(char **meta, int *offset)
{
struct trace *trace = trace_obj_get();
char uustr[RTE_UUID_STRLEN];
char *str = NULL;
int rc;
rte_uuid_unparse(trace->uuid, uustr, RTE_UUID_STRLEN);
rc = metadata_printf(&str,
"trace {\n"
" major = 1;\n"
" minor = 8;\n"
" uuid = \"%s\";\n"
" byte_order = %s;\n"
" packet.header := struct {\n"
" uint32_t magic;\n"
" uint8_t uuid[16];\n"
" };\n"
"};\n\n", uustr, is_be() ? "be" : "le");
return meta_copy(meta, offset, str, rc);
}
static int
meta_env_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"env {\n"
" dpdk_version = \"%s\";\n"
" tracer_name = \"dpdk\";\n"
"};\n\n", rte_version());
return meta_copy(meta, offset, str, rc);
}
static int
meta_clock_pass1_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"clock {\n"
" name = \"dpdk\";\n"
" freq = ");
return meta_copy(meta, offset, str, rc);
}
static int
meta_clock_pass2_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"%20"PRIu64";\n"
" offset_s =", 0);
return meta_copy(meta, offset, str, rc);
}
static int
meta_clock_pass3_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"%20"PRIu64";\n"
" offset =", 0);
return meta_copy(meta, offset, str, rc);
}
static int
meta_clock_pass4_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"%20"PRIu64";\n};\n\n"
"typealias integer {\n"
" size = 48; align = 1; signed = false;\n"
" map = clock.dpdk.value;\n"
"} := uint48_clock_dpdk_t;\n\n", 0);
return meta_copy(meta, offset, str, rc);
}
static int
meta_stream_emit(char **meta, int *offset)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"stream {\n"
" packet.context := struct {\n"
" uint32_t cpu_id;\n"
" string_bounded_t name[32];\n"
" };\n"
" event.header := struct {\n"
" uint48_clock_dpdk_t timestamp;\n"
" uint16_t id;\n"
" } align(64);\n"
"};\n\n");
return meta_copy(meta, offset, str, rc);
}
static int
meta_event_emit(char **meta, int *offset, struct trace_point *tp)
{
char *str = NULL;
int rc;
rc = metadata_printf(&str,
"event {\n"
" id = %d;\n"
" name = \"%s\";\n"
" fields := struct {\n"
"%s"
" };\n"
"};\n\n", trace_id_get(tp->handle), tp->name,
tp->ctf_field != NULL ? tp->ctf_field : "");
return meta_copy(meta, offset, str, rc);
}
int
trace_metadata_create(void)
{
struct trace_point_head *tp_list = trace_list_head_get();
struct trace *trace = trace_obj_get();
struct trace_point *tp;
int rc, offset = 0;
char *meta = NULL;
rc = meta_data_type_emit(&meta, &offset);
if (rc < 0)
goto fail;
rc = meta_header_emit(&meta, &offset);
if (rc < 0)
goto fail;
rc = meta_env_emit(&meta, &offset);
if (rc < 0)
goto fail;
rc = meta_clock_pass1_emit(&meta, &offset);
if (rc < 0)
goto fail;
trace->ctf_meta_offset_freq = offset;
rc = meta_clock_pass2_emit(&meta, &offset);
if (rc < 0)
goto fail;
trace->ctf_meta_offset_freq_off_s = offset;
rc = meta_clock_pass3_emit(&meta, &offset);
if (rc < 0)
goto fail;
trace->ctf_meta_offset_freq_off = offset;
rc = meta_clock_pass4_emit(&meta, &offset);
if (rc < 0)
goto fail;
rc = meta_stream_emit(&meta, &offset);
if (rc < 0)
goto fail;
STAILQ_FOREACH(tp, tp_list, next)
if (meta_event_emit(&meta, &offset, tp) < 0)
goto fail;
trace->ctf_meta = meta;
return 0;
fail:
free(meta);
return -EBADF;
}
void
trace_metadata_destroy(void)
{
struct trace *trace = trace_obj_get();
if (trace->ctf_meta) {
free(trace->ctf_meta);
trace->ctf_meta = NULL;
}
}
static void
meta_fix_freq(struct trace *trace, char *meta)
{
char *str;
int rc;
str = RTE_PTR_ADD(meta, trace->ctf_meta_offset_freq);
rc = sprintf(str, "%20"PRIu64"", rte_get_timer_hz());
str[rc] = ';';
}
static void
meta_fix_freq_offset(struct trace *trace, char *meta)
{
uint64_t uptime_tickes_floor, uptime_ticks, freq, uptime_sec;
uint64_t offset, offset_s;
char *str;
int rc;
uptime_ticks = trace->uptime_ticks &
((1ULL << __RTE_TRACE_EVENT_HEADER_ID_SHIFT) - 1);
freq = rte_get_tsc_hz();
uptime_tickes_floor = RTE_ALIGN_MUL_FLOOR(uptime_ticks, freq);
uptime_sec = uptime_tickes_floor / freq;
offset_s = trace->epoch_sec - uptime_sec;
offset = uptime_ticks - uptime_tickes_floor;
offset += trace->epoch_nsec * (freq / NSEC_PER_SEC);
str = RTE_PTR_ADD(meta, trace->ctf_meta_offset_freq_off_s);
rc = sprintf(str, "%20"PRIu64"", offset_s);
str[rc] = ';';
str = RTE_PTR_ADD(meta, trace->ctf_meta_offset_freq_off);
rc = sprintf(str, "%20"PRIu64"", offset);
str[rc] = ';';
}
static void
meta_fixup(struct trace *trace, char *meta)
{
meta_fix_freq(trace, meta);
meta_fix_freq_offset(trace, meta);
}
int
rte_trace_metadata_dump(FILE *f)
{
struct trace *trace = trace_obj_get();
char *ctf_meta = trace->ctf_meta;
int rc;
if (ctf_meta == NULL)
return -EINVAL;
if (!__atomic_load_n(&trace->ctf_fixup_done, __ATOMIC_SEQ_CST) &&
rte_get_timer_hz()) {
meta_fixup(trace, ctf_meta);
__atomic_store_n(&trace->ctf_fixup_done, 1, __ATOMIC_SEQ_CST);
}
rc = fprintf(f, "%s", ctf_meta);
return rc < 0 ? rc : 0;
}
char *trace_metadata_fixup_field(const char *field)
{
const char *ctf_reserved_words[] = {
"align",
"event",
};
unsigned int i;
char *out;
char *p;
/* reserved keywords */
for (i = 0; i < RTE_DIM(ctf_reserved_words); i++) {
if (strcmp(field, ctf_reserved_words[i]) != 0)
continue;
if (asprintf(&out, "_%s", ctf_reserved_words[i]) == -1)
out = NULL;
return out;
}
/* nothing to replace, return early */
if (strstr(field, ".") == NULL && strstr(field, "->") == NULL)
return NULL;
out = strdup(field);
if (out == NULL)
return NULL;
p = out;
while ((p = strstr(p, ".")) != NULL) {
p[0] = '_';
p++;
}
p = out;
while ((p = strstr(p, "->")) != NULL) {
p[0] = '_';
p++;
memmove(p, p + 1, strlen(p));
}
return out;
}

77
Programs/hugePage/common/eal_common_trace_points.c Normal file
View File

@@ -0,0 +1,77 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2020 Marvell International Ltd.
*/
#include "rte_trace_point_register.h"
#include "rte_eal_trace.h"
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_void,
lib.eal.generic.void)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_u64,
lib.eal.generic.u64)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_u32,
lib.eal.generic.u32)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_u16,
lib.eal.generic.u16)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_u8,
lib.eal.generic.u8)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_i64,
lib.eal.generic.i64)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_i32,
lib.eal.generic.i32)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_i16,
lib.eal.generic.i16)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_i8,
lib.eal.generic.i8)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_int,
lib.eal.generic.int)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_long,
lib.eal.generic.long)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_float,
lib.eal.generic.float)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_double,
lib.eal.generic.double)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_ptr,
lib.eal.generic.ptr)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_str,
lib.eal.generic.string)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_size_t,
lib.eal.generic.size_t)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_generic_func,
lib.eal.generic.func)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_alarm_set,
lib.eal.alarm.set)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_alarm_cancel,
lib.eal.alarm.cancel)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_mem_zmalloc,
lib.eal.mem.zmalloc)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_mem_malloc,
lib.eal.mem.malloc)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_mem_realloc,
lib.eal.mem.realloc)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_mem_free,
lib.eal.mem.free)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_memzone_reserve,
lib.eal.memzone.reserve)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_memzone_lookup,
lib.eal.memzone.lookup)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_memzone_free,
lib.eal.memzone.free)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_thread_remote_launch,
lib.eal.thread.remote.launch)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_thread_lcore_ready,
lib.eal.thread.lcore.ready)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_intr_callback_register,
lib.eal.intr.register)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_intr_callback_unregister,
lib.eal.intr.unregister)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_intr_enable,
lib.eal.intr.enable)
RTE_TRACE_POINT_REGISTER(rte_eal_trace_intr_disable,
lib.eal.intr.disable)

439
Programs/hugePage/common/eal_common_trace_utils.c Normal file
View File

@@ -0,0 +1,439 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2020 Marvell International Ltd.
*/
#include <fnmatch.h>
#include <pwd.h>
#include <sys/stat.h>
#include <time.h>
#include "rte_common.h"
#include "rte_errno.h"
#include "rte_string_fns.h"
#include "eal_filesystem.h"
#include "eal_trace.h"
const char *
trace_mode_to_string(enum rte_trace_mode mode)
{
switch (mode) {
case RTE_TRACE_MODE_OVERWRITE: return "overwrite";
case RTE_TRACE_MODE_DISCARD: return "discard";
default: return "unknown";
}
}
const char *
trace_area_to_string(enum trace_area_e area)
{
switch (area) {
case TRACE_AREA_HEAP: return "heap";
case TRACE_AREA_HUGEPAGE: return "hugepage";
default: return "unknown";
}
}
static bool
trace_entry_compare(const char *name)
{
struct trace_point_head *tp_list = trace_list_head_get();
struct trace_point *tp;
int count = 0;
STAILQ_FOREACH(tp, tp_list, next) {
if (strcmp(tp->name, name) == 0)
count++;
if (count > 1) {
trace_err("found duplicate entry %s", name);
rte_errno = EEXIST;
return true;
}
}
return false;
}
bool
trace_has_duplicate_entry(void)
{
struct trace_point_head *tp_list = trace_list_head_get();
struct trace_point *tp;
/* Is duplicate trace name registered */
STAILQ_FOREACH(tp, tp_list, next)
if (trace_entry_compare(tp->name))
return true;
return false;
}
void
trace_uuid_generate(void)
{
struct trace_point_head *tp_list = trace_list_head_get();
struct trace *trace = trace_obj_get();
struct trace_point *tp;
uint64_t sz_total = 0;
/* Go over the registered trace points to get total size of events */
STAILQ_FOREACH(tp, tp_list, next) {
const uint16_t sz = *tp->handle & __RTE_TRACE_FIELD_SIZE_MASK;
sz_total += sz;
}
rte_uuid_t uuid = RTE_UUID_INIT(sz_total, trace->nb_trace_points,
0x4370, 0x8f50, 0x222ddd514176ULL);
rte_uuid_copy(trace->uuid, uuid);
}
static int
trace_session_name_generate(char **trace_dir)
{
char date[sizeof("YYYY-mm-dd-AM-HH-MM-SS")];
struct tm *tm_result;
time_t tm;
tm = time(NULL);
if ((int)tm == -1)
goto fail;
tm_result = localtime(&tm);
if (tm_result == NULL)
goto fail;
if (strftime(date, sizeof(date), "%Y-%m-%d-%p-%I-%M-%S", tm_result) == 0) {
errno = ENOSPC;
goto fail;
}
if (asprintf(trace_dir, "%s-%s", eal_get_hugefile_prefix(), date) == -1)
goto fail;
return 0;
fail:
rte_errno = errno;
return -1;
}
static int
trace_dir_update(const char *str)
{
struct trace *trace = trace_obj_get();
char *dir;
int rc;
rc = asprintf(&dir, "%s%s", trace->dir != NULL ? trace->dir : "", str);
if (rc != -1) {
free(trace->dir);
trace->dir = dir;
}
return rc;
}
int
eal_trace_args_save(const char *val)
{
struct trace *trace = trace_obj_get();
struct trace_arg *arg = malloc(sizeof(*arg));
if (arg == NULL) {
trace_err("failed to allocate memory for %s", val);
return -ENOMEM;
}
arg->val = strdup(val);
if (arg->val == NULL) {
trace_err("failed to allocate memory for %s", val);
free(arg);
return -ENOMEM;
}
STAILQ_INSERT_TAIL(&trace->args, arg, next);
return 0;
}
void
eal_trace_args_free(void)
{
struct trace *trace = trace_obj_get();
struct trace_arg *arg;
while (!STAILQ_EMPTY(&trace->args)) {
arg = STAILQ_FIRST(&trace->args);
STAILQ_REMOVE_HEAD(&trace->args, next);
free(arg->val);
free(arg);
}
}
int
trace_args_apply(const char *arg)
{
if (rte_trace_regexp(arg, true) < 0) {
trace_err("cannot enable trace for %s", arg);
return -1;
}
return 0;
}
int
eal_trace_bufsz_args_save(char const *val)
{
struct trace *trace = trace_obj_get();
uint64_t bufsz;
bufsz = rte_str_to_size(val);
if (bufsz == 0) {
trace_err("buffer size cannot be zero");
return -EINVAL;
}
trace->buff_len = bufsz;
return 0;
}
void
trace_bufsz_args_apply(void)
{
struct trace *trace = trace_obj_get();
if (trace->buff_len == 0)
trace->buff_len = 1024 * 1024; /* 1MB */
}
int
eal_trace_mode_args_save(const char *val)
{
struct trace *trace = trace_obj_get();
size_t len = strlen(val);
unsigned long tmp;
char *pattern;
if (len == 0) {
trace_err("value is not provided with option");
return -EINVAL;
}
pattern = (char *)calloc(1, len + 2);
if (pattern == NULL) {
trace_err("fail to allocate memory");
return -ENOMEM;
}
sprintf(pattern, "%s*", val);
if (fnmatch(pattern, "overwrite", 0) == 0)
tmp = RTE_TRACE_MODE_OVERWRITE;
else if (fnmatch(pattern, "discard", 0) == 0)
tmp = RTE_TRACE_MODE_DISCARD;
else {
free(pattern);
return -EINVAL;
}
trace->mode = tmp;
free(pattern);
return 0;
}
int
eal_trace_dir_args_save(char const *val)
{
char *dir_path;
int rc;
if (asprintf(&dir_path, "%s/", val) == -1) {
trace_err("failed to copy directory: %s", strerror(errno));
return -ENOMEM;
}
rc = trace_dir_update(dir_path);
free(dir_path);
return rc;
}
int
trace_epoch_time_save(void)
{
struct trace *trace = trace_obj_get();
struct timespec epoch = { 0, 0 };
uint64_t avg, start, end;
start = rte_get_tsc_cycles();
if (clock_gettime(CLOCK_REALTIME, &epoch) < 0) {
trace_err("failed to get the epoch time");
return -1;
}
end = rte_get_tsc_cycles();
avg = (start + end) >> 1;
trace->epoch_sec = (uint64_t) epoch.tv_sec;
trace->epoch_nsec = (uint64_t) epoch.tv_nsec;
trace->uptime_ticks = avg;
return 0;
}
static int
trace_dir_default_path_get(char **dir_path)
{
struct passwd *pwd;
char *home_dir;
/* First check for shell environment variable */
home_dir = getenv("HOME");
if (home_dir == NULL) {
/* Fallback to password file entry */
pwd = getpwuid(getuid());
if (pwd == NULL)
return -EINVAL;
home_dir = pwd->pw_dir;
}
/* Append dpdk-traces to directory */
if (asprintf(dir_path, "%s/dpdk-traces/", home_dir) == -1)
return -ENOMEM;
return 0;
}
static int
trace_mkdir(void)
{
struct trace *trace = trace_obj_get();
static bool already_done;
char *session;
int rc;
if (already_done)
return 0;
if (trace->dir == NULL) {
char *dir_path;
rc = trace_dir_default_path_get(&dir_path);
if (rc < 0) {
trace_err("fail to get default path");
return rc;
}
rc = trace_dir_update(dir_path);
free(dir_path);
if (rc < 0)
return rc;
}
/* Create the path if it t exist, no "mkdir -p" available here */
rc = mkdir(trace->dir, 0700);
if (rc < 0 && errno != EEXIST) {
trace_err("mkdir %s failed [%s]", trace->dir, strerror(errno));
rte_errno = errno;
return -rte_errno;
}
rc = trace_session_name_generate(&session);
if (rc < 0)
return rc;
rc = trace_dir_update(session);
free(session);
if (rc < 0)
return rc;
rc = mkdir(trace->dir, 0700);
if (rc < 0) {
trace_err("mkdir %s failed [%s]", trace->dir, strerror(errno));
rte_errno = errno;
return -rte_errno;
}
RTE_LOG(INFO, EAL, "Trace dir: %s\n", trace->dir);
already_done = true;
return 0;
}
static int
trace_meta_save(struct trace *trace)
{
char file_name[PATH_MAX];
FILE *f;
int rc;
rc = snprintf(file_name, PATH_MAX, "%s/metadata", trace->dir);
if (rc < 0)
return rc;
f = fopen(file_name, "w");
if (f == NULL)
return -errno;
rc = rte_trace_metadata_dump(f);
if (fclose(f))
rc = -errno;
return rc;
}
static inline int
trace_file_sz(struct __rte_trace_header *hdr)
{
return sizeof(struct __rte_trace_stream_header) + hdr->offset;
}
static int
trace_mem_save(struct trace *trace, struct __rte_trace_header *hdr,
uint32_t cnt)
{
char file_name[PATH_MAX];
FILE *f;
int rc;
rc = snprintf(file_name, PATH_MAX, "%s/channel0_%d", trace->dir, cnt);
if (rc < 0)
return rc;
f = fopen(file_name, "w");
if (f == NULL)
return -errno;
rc = fwrite(&hdr->stream_header, trace_file_sz(hdr), 1, f);
rc = (rc == 1) ? 0 : -EACCES;
if (fclose(f))
rc = -errno;
return rc;
}
int
rte_trace_save(void)
{
struct trace *trace = trace_obj_get();
struct __rte_trace_header *header;
uint32_t count;
int rc = 0;
if (trace->nb_trace_mem_list == 0)
return rc;
rc = trace_mkdir();
if (rc < 0)
return rc;
rc = trace_meta_save(trace);
if (rc)
return rc;
rte_spinlock_lock(&trace->lock);
for (count = 0; count < trace->nb_trace_mem_list; count++) {
header = trace->lcore_meta[count].mem;
rc = trace_mem_save(trace, header, count);
if (rc)
break;
}
rte_spinlock_unlock(&trace->lock);
return rc;
}

167
Programs/hugePage/common/eal_common_uuid.c Normal file
View File

@@ -0,0 +1,167 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C) 1996, 1997 Theodore Ts'o.
*/
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#include <ctype.h>
#include "rte_uuid.h"
/* UUID packed form */
struct uuid {
uint32_t time_low;
uint16_t time_mid;
uint16_t time_hi_and_version;
uint16_t clock_seq;
uint8_t node[6];
};
static void uuid_pack(const struct uuid *uu, rte_uuid_t ptr)
{
uint32_t tmp;
uint8_t *out = ptr;
tmp = uu->time_low;
out[3] = (uint8_t) tmp;
tmp >>= 8;
out[2] = (uint8_t) tmp;
tmp >>= 8;
out[1] = (uint8_t) tmp;
tmp >>= 8;
out[0] = (uint8_t) tmp;
tmp = uu->time_mid;
out[5] = (uint8_t) tmp;
tmp >>= 8;
out[4] = (uint8_t) tmp;
tmp = uu->time_hi_and_version;
out[7] = (uint8_t) tmp;
tmp >>= 8;
out[6] = (uint8_t) tmp;
tmp = uu->clock_seq;
out[9] = (uint8_t) tmp;
tmp >>= 8;
out[8] = (uint8_t) tmp;
memcpy(out+10, uu->node, 6);
}
static void uuid_unpack(const rte_uuid_t in, struct uuid *uu)
{
const uint8_t *ptr = in;
uint32_t tmp;
tmp = *ptr++;
tmp = (tmp << 8) | *ptr++;
tmp = (tmp << 8) | *ptr++;
tmp = (tmp << 8) | *ptr++;
uu->time_low = tmp;
tmp = *ptr++;
tmp = (tmp << 8) | *ptr++;
uu->time_mid = tmp;
tmp = *ptr++;
tmp = (tmp << 8) | *ptr++;
uu->time_hi_and_version = tmp;
tmp = *ptr++;
tmp = (tmp << 8) | *ptr++;
uu->clock_seq = tmp;
memcpy(uu->node, ptr, 6);
}
bool rte_uuid_is_null(const rte_uuid_t uu)
{
const uint8_t *cp = uu;
int i;
for (i = 0; i < 16; i++)
if (*cp++)
return false;
return true;
}
/*
* rte_uuid_compare() - compare two UUIDs.
*/
int rte_uuid_compare(const rte_uuid_t uu1, const rte_uuid_t uu2)
{
struct uuid uuid1, uuid2;
uuid_unpack(uu1, &uuid1);
uuid_unpack(uu2, &uuid2);
#define UUCMP(u1, u2) \
do { if (u1 != u2) return (u1 < u2) ? -1 : 1; } while (0)
UUCMP(uuid1.time_low, uuid2.time_low);
UUCMP(uuid1.time_mid, uuid2.time_mid);
UUCMP(uuid1.time_hi_and_version, uuid2.time_hi_and_version);
UUCMP(uuid1.clock_seq, uuid2.clock_seq);
#undef UUCMP
return memcmp(uuid1.node, uuid2.node, 6);
}
int rte_uuid_parse(const char *in, rte_uuid_t uu)
{
struct uuid uuid;
int i;
const char *cp;
char buf[3];
if (strlen(in) != 36)
return -1;
for (i = 0, cp = in; i <= 36; i++, cp++) {
if ((i == 8) || (i == 13) || (i == 18) ||
(i == 23)) {
if (*cp == '-')
continue;
else
return -1;
}
if (i == 36)
if (*cp == 0)
continue;
if (!isxdigit(*cp))
return -1;
}
uuid.time_low = strtoul(in, NULL, 16);
uuid.time_mid = strtoul(in+9, NULL, 16);
uuid.time_hi_and_version = strtoul(in+14, NULL, 16);
uuid.clock_seq = strtoul(in+19, NULL, 16);
cp = in+24;
buf[2] = 0;
for (i = 0; i < 6; i++) {
buf[0] = *cp++;
buf[1] = *cp++;
uuid.node[i] = strtoul(buf, NULL, 16);
}
uuid_pack(&uuid, uu);
return 0;
}
void rte_uuid_unparse(const rte_uuid_t uu, char *out, size_t len)
{
struct uuid uuid;
uuid_unpack(uu, &uuid);
snprintf(out, len,
"%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x",
uuid.time_low, uuid.time_mid, uuid.time_hi_and_version,
uuid.clock_seq >> 8, uuid.clock_seq & 0xFF,
uuid.node[0], uuid.node[1], uuid.node[2],
uuid.node[3], uuid.node[4], uuid.node[5]);
}

107
Programs/hugePage/common/eal_filesystem.h Normal file
View File

@@ -0,0 +1,107 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2018 Intel Corporation
*/
/**
* @file
* Stores functions and path defines for files and directories
* on the filesystem for Linux, that are used by the Linux EAL.
*/
#ifndef EAL_FILESYSTEM_H
#define EAL_FILESYSTEM_H
/** Path of rte config file. */
#include <stdint.h>
#include <limits.h>
#include <unistd.h>
#include <stdlib.h>
#include "rte_string_fns.h"
#include "eal_internal_cfg.h"
/* sets up platform-specific runtime data dir */
int
eal_create_runtime_dir(void);
int
eal_clean_runtime_dir(void);
/** Function to return hugefile prefix that's currently set up */
const char *
eal_get_hugefile_prefix(void);
#define RUNTIME_CONFIG_FNAME "config"
static inline const char *
eal_runtime_config_path(void)
{
static char buffer[PATH_MAX]; /* static so auto-zeroed */
snprintf(buffer, sizeof(buffer), "%s/%s", rte_eal_get_runtime_dir(),
RUNTIME_CONFIG_FNAME);
return buffer;
}
/** Path of primary/secondary communication unix socket file. */
#define MP_SOCKET_FNAME "mp_socket"
static inline const char *
eal_mp_socket_path(void)
{
static char buffer[PATH_MAX]; /* static so auto-zeroed */
snprintf(buffer, sizeof(buffer), "%s/%s", rte_eal_get_runtime_dir(),
MP_SOCKET_FNAME);
return buffer;
}
#define FBARRAY_NAME_FMT "%s/fbarray_%s"
static inline const char *
eal_get_fbarray_path(char *buffer, size_t buflen, const char *name) {
snprintf(buffer, buflen, FBARRAY_NAME_FMT, rte_eal_get_runtime_dir(),
name);
return buffer;
}
/** Path of hugepage info file. */
#define HUGEPAGE_INFO_FNAME "hugepage_info"
static inline const char *
eal_hugepage_info_path(void)
{
static char buffer[PATH_MAX]; /* static so auto-zeroed */
snprintf(buffer, sizeof(buffer), "%s/%s", rte_eal_get_runtime_dir(),
HUGEPAGE_INFO_FNAME);
return buffer;
}
/** Path of hugepage data file. */
#define HUGEPAGE_DATA_FNAME "hugepage_data"
static inline const char *
eal_hugepage_data_path(void)
{
static char buffer[PATH_MAX]; /* static so auto-zeroed */
snprintf(buffer, sizeof(buffer), "%s/%s", rte_eal_get_runtime_dir(),
HUGEPAGE_DATA_FNAME);
return buffer;
}
/** String format for hugepage map files. */
#define HUGEFILE_FMT "%s/%smap_%d"
static inline const char *
eal_get_hugefile_path(char *buffer, size_t buflen, const char *hugedir, int f_id)
{
snprintf(buffer, buflen, HUGEFILE_FMT, hugedir,
eal_get_hugefile_prefix(), f_id);
return buffer;
}
/** define the default filename prefix for the %s values above */
#define HUGEFILE_PREFIX_DEFAULT "rte"
/** Function to read a single numeric value from a file on the filesystem.
* Used to read information from files on /sys */
int eal_parse_sysfs_value(const char *filename, unsigned long *val);
#endif /* EAL_FILESYSTEM_H */

32
Programs/hugePage/common/eal_firmware.h Normal file
View File

@@ -0,0 +1,32 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2021 Red Hat, Inc.
*/
#ifndef _EAL_FIRMWARE_H_
#define _EAL_FIRMWARE_H_
#include <sys/types.h>
#include "rte_compat.h"
/**
* Load a firmware in a dynamically allocated buffer, dealing with compressed
* files if libarchive is available.
*
* @param[in] name
* Firmware filename to load.
* @param[out] buf
* Buffer allocated by this function. If this function succeeds, the
* caller is responsible for calling free() on this buffer.
* @param[out] bufsz
* Size of the data in the buffer.
*
* @return
* 0 if successful.
* Negative otherwise, buf and bufsize contents are invalid.
*/
__rte_internal
int
rte_firmware_read(const char *name, void **buf, size_t *bufsz);
#endif /* _EAL_FIRMWARE_H_ */

40
Programs/hugePage/common/eal_hugepages.h Normal file
View File

@@ -0,0 +1,40 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef EAL_HUGEPAGES_H
#define EAL_HUGEPAGES_H
#include <stddef.h>
#include <stdint.h>
#include <limits.h>
#define MAX_HUGEPAGE_PATH PATH_MAX
/**
* Structure used to store information about hugepages that we mapped
* through the files in hugetlbfs.
*/
struct hugepage_file {
void *orig_va; /**< virtual addr of first mmap() */
void *final_va; /**< virtual addr of 2nd mmap() */
uint64_t physaddr; /**< physical addr */
size_t size; /**< the page size */
int socket_id; /**< NUMA socket ID */
int file_id; /**< the '%d' in HUGEFILE_FMT */
char filepath[MAX_HUGEPAGE_PATH]; /**< path to backing file on filesystem */
};
/**
* Read the information on what hugepages are available for the EAL to use,
* clearing out any unused ones.
*/
int eal_hugepage_info_init(void);
/**
* Read whatever information primary process has shared about hugepages into
* secondary process.
*/
int eal_hugepage_info_read(void);
#endif /* EAL_HUGEPAGES_H */

110
Programs/hugePage/common/eal_internal_cfg.h Normal file
View File

@@ -0,0 +1,110 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
/**
* @file
* Holds the structures for the eal internal configuration
*/
#ifndef EAL_INTERNAL_CFG_H
#define EAL_INTERNAL_CFG_H
#include "rte_eal.h"
// #include "rte_os_shim.h"
#include "rte_pci_dev_feature_defs.h"
#include "eal_thread.h"
#if defined(RTE_ARCH_ARM)
#define MAX_HUGEPAGE_SIZES 4 /**< support up to 4 page sizes */
#else
#define MAX_HUGEPAGE_SIZES 3 /**< support up to 3 page sizes */
#endif
/*
* internal configuration structure for the number, size and
* mount points of hugepages
*/
struct hugepage_info {
uint64_t hugepage_sz; /**< size of a huge page */
char hugedir[PATH_MAX]; /**< dir where hugetlbfs is mounted */
uint32_t num_pages[RTE_MAX_NUMA_NODES];
/**< number of hugepages of that size on each socket */
int lock_descriptor; /**< file descriptor for hugepage dir */
};
struct simd_bitwidth {
bool forced;
/**< flag indicating if bitwidth is forced and can't be modified */
uint16_t bitwidth; /**< bitwidth value */
};
/** Hugepage backing files discipline. */
struct hugepage_file_discipline {
/** Unlink files before mapping them to leave no trace in hugetlbfs. */
bool unlink_before_mapping;
/** Unlink existing files at startup, re-create them before mapping. */
bool unlink_existing;
};
/**
* internal configuration
*/
struct internal_config {
volatile size_t memory; /**< amount of asked memory */
volatile unsigned force_nchannel; /**< force number of channels */
volatile unsigned force_nrank; /**< force number of ranks */
volatile unsigned no_hugetlbfs; /**< true to disable hugetlbfs */
struct hugepage_file_discipline hugepage_file;
volatile unsigned no_pci; /**< true to disable PCI */
volatile unsigned no_hpet; /**< true to disable HPET */
volatile unsigned vmware_tsc_map; /**< true to use VMware TSC mapping
* instead of native TSC */
volatile unsigned no_shconf; /**< true if there is no shared config */
volatile unsigned in_memory;
/**< true if DPDK should operate entirely in-memory and not create any
* shared files or runtime data.
*/
volatile unsigned create_uio_dev; /**< true to create /dev/uioX devices */
volatile enum rte_proc_type_t process_type; /**< multi-process proc type */
/** true to try allocating memory on specific sockets */
volatile unsigned force_sockets;
volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */
volatile unsigned force_socket_limits;
volatile uint64_t socket_limit[RTE_MAX_NUMA_NODES]; /**< limit amount of memory per socket */
uintptr_t base_virtaddr; /**< base address to try and reserve memory from */
volatile unsigned legacy_mem;
/**< true to enable legacy memory behavior (no dynamic allocation,
* IOVA-contiguous segments).
*/
volatile unsigned match_allocations;
/**< true to free hugepages exactly as allocated */
volatile unsigned single_file_segments;
/**< true if storing all pages within single files (per-page-size,
* per-node) non-legacy mode only.
*/
volatile int syslog_facility; /**< facility passed to openlog() */
/** default interrupt mode for VFIO */
volatile enum rte_intr_mode vfio_intr_mode;
/** the shared VF token for VFIO-PCI bound PF and VFs devices */
rte_uuid_t vfio_vf_token;
char *hugefile_prefix; /**< the base filename of hugetlbfs files */
char *hugepage_dir; /**< specific hugetlbfs directory to use */
char *user_mbuf_pool_ops_name;
/**< user defined mbuf pool ops name */
unsigned num_hugepage_sizes; /**< how many sizes on this system */
struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
enum rte_iova_mode iova_mode ; /**< Set IOVA mode on this system */
rte_cpuset_t ctrl_cpuset; /**< cpuset for ctrl threads */
volatile unsigned int init_complete;
/**< indicates whether EAL has completed initialization */
unsigned int no_telemetry; /**< true to disable Telemetry */
struct simd_bitwidth max_simd_bitwidth;
/**< max simd bitwidth path to use */
size_t huge_worker_stack_size; /**< worker thread stack size */
};
void eal_reset_internal_config(struct internal_config *internal_cfg);
#endif /* EAL_INTERNAL_CFG_H */

30
Programs/hugePage/common/eal_interrupts.h Normal file
View File

@@ -0,0 +1,30 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef EAL_INTERRUPTS_H
#define EAL_INTERRUPTS_H
struct rte_intr_handle {
RTE_STD_C11
union {
struct {
int dev_fd; /**< VFIO/UIO cfg device file descriptor */
int fd; /**< interrupt event file descriptor */
};
void *windows_handle; /**< device driver handle */
};
uint32_t alloc_flags; /**< flags passed at allocation */
enum rte_intr_handle_type type; /**< handle type */
uint32_t max_intr; /**< max interrupt requested */
uint32_t nb_efd; /**< number of available efd(event fd) */
uint8_t efd_counter_size; /**< size of efd counter, used for vdev */
uint16_t nb_intr;
/**< Max vector count, default RTE_MAX_RXTX_INTR_VEC_ID */
int *efds; /**< intr vectors/efds mapping */
struct rte_epoll_event *elist; /**< intr vector epoll event */
uint16_t vec_list_size;
int *intr_vec; /**< intr vector number array */
};
#endif /* EAL_INTERRUPTS_H */

32
Programs/hugePage/common/eal_log.h Normal file
View File

@@ -0,0 +1,32 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2021 Mellanox Technologies, Ltd
*/
#ifndef EAL_LOG_H
#define EAL_LOG_H
#include <stdio.h>
#include <stdint.h>
/*
* Initialize the default log stream.
*/
int eal_log_init(const char *id, int facility);
/*
* Determine where log data is written when no call to rte_openlog_stream.
*/
void eal_log_set_default(FILE *default_log);
/*
* Save a log option for later.
*/
int eal_log_save_regexp(const char *regexp, uint32_t level);
int eal_log_save_pattern(const char *pattern, uint32_t level);
/*
* Convert log level to string.
*/
const char *eal_log_level2str(uint32_t level);
#endif /* EAL_LOG_H */

99
Programs/hugePage/common/eal_memalloc.h Normal file
View File

@@ -0,0 +1,99 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017-2018 Intel Corporation
*/
#ifndef EAL_MEMALLOC_H
#define EAL_MEMALLOC_H
#include <stdbool.h>
#include "rte_memory.h"
/*
* Allocate segment of specified page size.
*/
struct rte_memseg *
eal_memalloc_alloc_seg(size_t page_sz, int socket);
/*
* Allocate `n_segs` segments.
*
* Note: `ms` can be NULL.
*
* Note: it is possible to request best-effort allocation by setting `exact` to
* `false`, in which case allocator will return however many pages it managed to
* allocate successfully.
*/
int
eal_memalloc_alloc_seg_bulk(struct rte_memseg **ms, int n_segs, size_t page_sz,
int socket, bool exact);
/*
* Deallocate segment
*/
int
eal_memalloc_free_seg(struct rte_memseg *ms);
/*
* Deallocate `n_segs` segments. Returns 0 on successful deallocation of all
* segments, returns -1 on error. Any segments that could have been deallocated,
* will be deallocated even in case of error.
*/
int
eal_memalloc_free_seg_bulk(struct rte_memseg **ms, int n_segs);
/*
* Check if memory pointed to by `start` and of `length` that resides in
* memseg list `msl` is IOVA-contiguous.
*/
bool
eal_memalloc_is_contig(const struct rte_memseg_list *msl, void *start,
size_t len);
/* synchronize local memory map to primary process */
int
eal_memalloc_sync_with_primary(void);
int
eal_memalloc_mem_event_callback_register(const char *name,
rte_mem_event_callback_t clb, void *arg);
int
eal_memalloc_mem_event_callback_unregister(const char *name, void *arg);
void
eal_memalloc_mem_event_notify(enum rte_mem_event event, const void *start,
size_t len);
int
eal_memalloc_mem_alloc_validator_register(const char *name,
rte_mem_alloc_validator_t clb, int socket_id, size_t limit);
int
eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id);
int
eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len);
/* returns fd or -errno */
int
eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
/* returns 0 or -errno */
int
eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
/* returns 0 or -errno */
int
eal_memalloc_set_seg_list_fd(int list_idx, int fd);
int
eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset);
int
eal_memalloc_init(void);
int
eal_memalloc_cleanup(void);
#endif /* EAL_MEMALLOC_H */

100
Programs/hugePage/common/eal_memcfg.h Normal file
View File

@@ -0,0 +1,100 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2019 Intel Corporation
*/
#ifndef EAL_MEMCFG_H
#define EAL_MEMCFG_H
#include "rte_memory.h"
#include "rte_memzone.h"
#include "rte_pause.h"
#include "rte_spinlock.h"
#include "rte_rwlock.h"
#include "rte_tailq.h"
#include "malloc_heap.h"
/**
* Memory configuration shared across multiple processes.
*/
struct rte_mem_config {
volatile uint32_t magic; /**< Magic number - sanity check. */
uint32_t version;
/**< Prevent secondary processes using different DPDK versions. */
/* memory topology */
uint32_t nchannel; /**< Number of channels (0 if unknown). */
uint32_t nrank; /**< Number of ranks (0 if unknown). */
/**
* current lock nest order
* - qlock->mlock (ring/hash/lpm)
* - mplock->qlock->mlock (mempool)
* Notice:
* *ALWAYS* obtain qlock first if having to obtain both qlock and mlock
*/
rte_rwlock_t mlock; /**< used by memzones for thread safety. */
rte_rwlock_t qlock; /**< used by tailqs for thread safety. */
rte_rwlock_t mplock; /**< used by mempool library for thread safety. */
rte_spinlock_t tlock; /**< used by timer library for thread safety. */
rte_rwlock_t memory_hotplug_lock;
/**< Indicates whether memory hotplug request is in progress. */
uint8_t mp_status; /**< Multiprocess status. */
/* memory segments and zones */
struct rte_fbarray memzones; /**< Memzone descriptors. */
struct rte_memseg_list memsegs[RTE_MAX_MEMSEG_LISTS];
/**< List of dynamic arrays holding memsegs */
struct rte_tailq_head tailq_head[RTE_MAX_TAILQ];
/**< Tailqs for objects */
struct malloc_heap malloc_heaps[RTE_MAX_HEAPS];
/**< DPDK malloc heaps */
int next_socket_id; /**< Next socket ID for external malloc heap */
/* rte_mem_config has to be mapped at the exact same address in all
* processes, so we need to store it.
*/
uint64_t mem_cfg_addr; /**< Address of this structure in memory. */
/* Primary and secondary processes cannot run with different legacy or
* single file segments options, so to avoid having to specify these
* options to all processes, store them in shared config and update the
* internal config at init time.
*/
uint32_t legacy_mem; /**< stored legacy mem parameter. */
uint32_t single_file_segments;
/**< stored single file segments parameter. */
uint64_t tsc_hz;
/**< TSC rate */
uint8_t dma_maskbits; /**< Keeps the more restricted dma mask. */
};
/* update internal config from shared mem config */
void
eal_mcfg_update_internal(void);
/* update shared mem config from internal config */
void
eal_mcfg_update_from_internal(void);
/* wait until primary process initialization is complete */
void
eal_mcfg_wait_complete(void);
/* check if DPDK version of current process matches one stored in the config */
int
eal_mcfg_check_version(void);
/* set mem config as complete */
void
eal_mcfg_complete(void);
#endif /* EAL_MEMCFG_H */

112
Programs/hugePage/common/eal_options.h Normal file
View File

@@ -0,0 +1,112 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2014 6WIND S.A.
*/
#ifndef EAL_OPTIONS_H
#define EAL_OPTIONS_H
#include "getopt.h"
struct rte_tel_data;
enum {
/* long options mapped to a short option */
#define OPT_HELP "help"
OPT_HELP_NUM = 'h',
#define OPT_DEV_ALLOW "allow"
OPT_DEV_ALLOW_NUM = 'a',
#define OPT_DEV_BLOCK "block"
OPT_DEV_BLOCK_NUM = 'b',
/* first long only option value must be >= 256, so that we won't
* conflict with short options */
OPT_LONG_MIN_NUM = 256,
#define OPT_BASE_VIRTADDR "base-virtaddr"
OPT_BASE_VIRTADDR_NUM,
#define OPT_CREATE_UIO_DEV "create-uio-dev"
OPT_CREATE_UIO_DEV_NUM,
#define OPT_FILE_PREFIX "file-prefix"
OPT_FILE_PREFIX_NUM,
#define OPT_HUGE_DIR "huge-dir"
OPT_HUGE_DIR_NUM,
#define OPT_HUGE_UNLINK "huge-unlink"
OPT_HUGE_UNLINK_NUM,
#define OPT_LCORES "lcores"
OPT_LCORES_NUM,
#define OPT_LOG_LEVEL "log-level"
OPT_LOG_LEVEL_NUM,
#define OPT_TRACE "trace"
OPT_TRACE_NUM,
#define OPT_TRACE_DIR "trace-dir"
OPT_TRACE_DIR_NUM,
#define OPT_TRACE_BUF_SIZE "trace-bufsz"
OPT_TRACE_BUF_SIZE_NUM,
#define OPT_TRACE_MODE "trace-mode"
OPT_TRACE_MODE_NUM,
#define OPT_MAIN_LCORE "main-lcore"
OPT_MAIN_LCORE_NUM,
#define OPT_MBUF_POOL_OPS_NAME "mbuf-pool-ops-name"
OPT_MBUF_POOL_OPS_NAME_NUM,
#define OPT_PROC_TYPE "proc-type"
OPT_PROC_TYPE_NUM,
#define OPT_NO_HPET "no-hpet"
OPT_NO_HPET_NUM,
#define OPT_NO_HUGE "no-huge"
OPT_NO_HUGE_NUM,
#define OPT_NO_PCI "no-pci"
OPT_NO_PCI_NUM,
#define OPT_NO_SHCONF "no-shconf"
OPT_NO_SHCONF_NUM,
#define OPT_IN_MEMORY "in-memory"
OPT_IN_MEMORY_NUM,
#define OPT_SOCKET_MEM "socket-mem"
OPT_SOCKET_MEM_NUM,
#define OPT_SOCKET_LIMIT "socket-limit"
OPT_SOCKET_LIMIT_NUM,
#define OPT_SYSLOG "syslog"
OPT_SYSLOG_NUM,
#define OPT_VDEV "vdev"
OPT_VDEV_NUM,
#define OPT_VFIO_INTR "vfio-intr"
OPT_VFIO_INTR_NUM,
#define OPT_VFIO_VF_TOKEN "vfio-vf-token"
OPT_VFIO_VF_TOKEN_NUM,
#define OPT_VMWARE_TSC_MAP "vmware-tsc-map"
OPT_VMWARE_TSC_MAP_NUM,
#define OPT_LEGACY_MEM "legacy-mem"
OPT_LEGACY_MEM_NUM,
#define OPT_SINGLE_FILE_SEGMENTS "single-file-segments"
OPT_SINGLE_FILE_SEGMENTS_NUM,
#define OPT_IOVA_MODE "iova-mode"
OPT_IOVA_MODE_NUM,
#define OPT_MATCH_ALLOCATIONS "match-allocations"
OPT_MATCH_ALLOCATIONS_NUM,
#define OPT_TELEMETRY "telemetry"
OPT_TELEMETRY_NUM,
#define OPT_NO_TELEMETRY "no-telemetry"
OPT_NO_TELEMETRY_NUM,
#define OPT_FORCE_MAX_SIMD_BITWIDTH "force-max-simd-bitwidth"
OPT_FORCE_MAX_SIMD_BITWIDTH_NUM,
#define OPT_HUGE_WORKER_STACK "huge-worker-stack"
OPT_HUGE_WORKER_STACK_NUM,
OPT_LONG_MAX_NUM
};
extern const char eal_short_options[];
extern const struct option eal_long_options[];
int eal_parse_common_option(int opt, const char *argv,
struct internal_config *conf);
int eal_option_device_parse(void);
int eal_adjust_config(struct internal_config *internal_cfg);
int eal_cleanup_config(struct internal_config *internal_cfg);
int eal_check_common_options(struct internal_config *internal_cfg);
void eal_common_usage(void);
enum rte_proc_type_t eal_proc_type_detect(void);
int eal_plugins_init(void);
int eal_save_args(int argc, char **argv);
int handle_eal_info_request(const char *cmd, const char *params __rte_unused,
struct rte_tel_data *d);
#endif /* EAL_OPTIONS_H */

760
Programs/hugePage/common/eal_private.h Normal file
View File

@@ -0,0 +1,760 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2018 Intel Corporation
*/
#ifndef _EAL_PRIVATE_H_
#define _EAL_PRIVATE_H_
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <sys/queue.h>
#include "dev_driver.h"
#include "rte_lcore.h"
#include "rte_memory.h"
#include "eal_internal_cfg.h"
/**
* Structure storing internal configuration (per-lcore)
*/
struct lcore_config {
pthread_t thread_id; /**< pthread identifier */
int pipe_main2worker[2]; /**< communication pipe with main */
int pipe_worker2main[2]; /**< communication pipe with main */
lcore_function_t * volatile f; /**< function to call */
void * volatile arg; /**< argument of function */
volatile int ret; /**< return value of function */
volatile enum rte_lcore_state_t state; /**< lcore state */
unsigned int socket_id; /**< physical socket id for this lcore */
unsigned int core_id; /**< core number on socket for this lcore */
int core_index; /**< relative index, starting from 0 */
uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */
rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */
};
extern struct lcore_config lcore_config[RTE_MAX_LCORE];
/**
* The global RTE configuration structure.
*/
struct rte_config {
uint32_t main_lcore; /**< Id of the main lcore */
uint32_t lcore_count; /**< Number of available logical cores. */
uint32_t numa_node_count; /**< Number of detected NUMA nodes. */
uint32_t numa_nodes[RTE_MAX_NUMA_NODES]; /**< List of detected NUMA nodes. */
uint32_t service_lcore_count;/**< Number of available service cores. */
enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
/** Primary or secondary configuration */
enum rte_proc_type_t process_type;
/** PA or VA mapping mode */
enum rte_iova_mode iova_mode;
/**
* Pointer to memory configuration, which may be shared across multiple
* DPDK instances
*/
struct rte_mem_config *mem_config;
#ifndef RTE_ARCH_ARM_PURECAP_HACK
} __rte_packed;
#else
};
#endif
/**
* Get the global configuration structure.
*
* @return
* A pointer to the global configuration structure.
*/
struct rte_config *rte_eal_get_configuration(void);
/**
* Initialize the memzone subsystem (private to eal).
*
* @return
* - 0 on success
* - Negative on error
*/
int rte_eal_memzone_init(void);
/**
* Fill configuration with number of physical and logical processors
*
* This function is private to EAL.
*
* Parse /proc/cpuinfo to get the number of physical and logical
* processors on the machine.
*
* @return
* 0 on success, negative on error
*/
int rte_eal_cpu_init(void);
/**
* Create memseg lists
*
* This function is private to EAL.
*
* Preallocate virtual memory.
*
* @return
* 0 on success, negative on error
*/
int rte_eal_memseg_init(void);
/**
* Map memory
*
* This function is private to EAL.
*
* Fill configuration structure with these infos, and return 0 on success.
*
* @return
* 0 on success, negative on error
*/
int rte_eal_memory_init(void);
/**
* Configure timers
*
* This function is private to EAL.
*
* Mmap memory areas used by HPET (high precision event timer) that will
* provide our time reference, and configure the TSC frequency also for it
* to be used as a reference.
*
* @return
* 0 on success, negative on error
*/
int rte_eal_timer_init(void);
/**
* Init tail queues for non-EAL library structures. This is to allow
* the rings, mempools, etc. lists to be shared among multiple processes
*
* This function is private to EAL
*
* @return
* 0 on success, negative on error
*/
int rte_eal_tailqs_init(void);
/**
* Init interrupt handling.
*
* This function is private to EAL.
*
* @return
* 0 on success, negative on error
*/
int rte_eal_intr_init(void);
/**
* Close the default log stream
*
* This function is private to EAL.
*/
void rte_eal_log_cleanup(void);
/**
* Init alarm mechanism. This is to allow a callback be called after
* specific time.
*
* This function is private to EAL.
*
* @return
* 0 on success, negative on error
*/
int rte_eal_alarm_init(void);
/**
* Alarm mechanism cleanup.
*
* This function is private to EAL.
*
* @return
* 0 on success, negative on error
*/
void rte_eal_alarm_cleanup(void);
/**
* Function is to check if the kernel module(like, vfio, vfio_iommu_type1,
* etc.) loaded.
*
* @param module_name
* The module's name which need to be checked
*
* @return
* -1 means some error happens(NULL pointer or open failure)
* 0 means the module not loaded
* 1 means the module loaded
*/
int rte_eal_check_module(const char *module_name);
/**
* Memory reservation flags.
*/
enum eal_mem_reserve_flags {
/**
* Reserve hugepages. May be unsupported by some platforms.
*/
EAL_RESERVE_HUGEPAGES = 1 << 0,
/**
* Force reserving memory at the requested address.
* This can be a destructive action depending on the implementation.
*
* @see RTE_MAP_FORCE_ADDRESS for description of possible consequences
* (although implementations are not required to use it).
*/
EAL_RESERVE_FORCE_ADDRESS = 1 << 1
};
/**
* Get virtual area of specified size from the OS.
*
* This function is private to the EAL.
*
* @param requested_addr
* Address where to request address space.
* @param size
* Size of requested area.
* @param page_sz
* Page size on which to align requested virtual area.
* @param flags
* EAL_VIRTUAL_AREA_* flags.
* @param reserve_flags
* Extra flags passed directly to eal_mem_reserve().
*
* @return
* Virtual area address if successful.
* NULL if unsuccessful.
*/
#define EAL_VIRTUAL_AREA_ADDR_IS_HINT (1 << 0)
/**< don't fail if cannot get exact requested address. */
#define EAL_VIRTUAL_AREA_ALLOW_SHRINK (1 << 1)
/**< try getting smaller sized (decrement by page size) virtual areas if cannot
* get area of requested size.
*/
#define EAL_VIRTUAL_AREA_UNMAP (1 << 2)
/**< immediately unmap reserved virtual area. */
void *
eal_get_virtual_area(void *requested_addr, size_t *size,
size_t page_sz, int flags, int reserve_flags);
/**
* Initialize a memory segment list and create its backing storage.
*
* @param msl
* Memory segment list to be filled.
* @param name
* Name for the backing storage.
* @param page_sz
* Size of segment pages in the MSL.
* @param n_segs
* Number of segments.
* @param socket_id
* Socket ID. Must not be SOCKET_ID_ANY.
* @param heap
* Mark MSL as pointing to a heap.
* @return
* 0 on success, (-1) on failure and rte_errno is set.
*/
int
eal_memseg_list_init_named(struct rte_memseg_list *msl, const char *name,
uint64_t page_sz, int n_segs, int socket_id, bool heap);
/**
* Initialize memory segment list and create its backing storage
* with a name corresponding to MSL parameters.
*
* @param type_msl_idx
* Index of the MSL among other MSLs of the same socket and page size.
*
* @see eal_memseg_list_init_named for remaining parameters description.
*/
int
eal_memseg_list_init(struct rte_memseg_list *msl, uint64_t page_sz,
int n_segs, int socket_id, int type_msl_idx, bool heap);
/**
* Reserve VA space for a memory segment list
* previously initialized with eal_memseg_list_init().
*
* @param msl
* Initialized memory segment list with page size defined.
* @param reserve_flags
* Extra memory reservation flags. Can be 0 if unnecessary.
* @return
* 0 on success, (-1) on failure and rte_errno is set.
*/
int
eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags);
/**
* Populate MSL, each segment is one page long.
*
* @param msl
* Initialized memory segment list with page size defined.
* @param addr
* Starting address of list segments.
* @param n_segs
* Number of segments to populate.
*/
void
eal_memseg_list_populate(struct rte_memseg_list *msl, void *addr, int n_segs);
/**
* Distribute available memory between MSLs.
*
* @return
* 0 on success, (-1) on failure.
*/
int
eal_dynmem_memseg_lists_init(void);
/**
* Preallocate hugepages for dynamic allocation.
*
* @return
* 0 on success, (-1) on failure.
*/
int
eal_dynmem_hugepage_init(void);
/**
* Given the list of hugepage sizes and the number of pages thereof,
* calculate the best number of pages of each size to fulfill the request
* for RAM on each NUMA node.
*
* @param memory
* Amounts of memory requested for each NUMA node of RTE_MAX_NUMA_NODES.
* @param hp_info
* Information about hugepages of different size.
* @param hp_used
* Receives information about used hugepages of each size.
* @param num_hp_info
* Number of elements in hp_info and hp_used.
* @return
* 0 on success, (-1) on failure.
*/
int
eal_dynmem_calc_num_pages_per_socket(
uint64_t *memory, struct hugepage_info *hp_info,
struct hugepage_info *hp_used, unsigned int num_hp_info);
/**
* Get cpu core_id.
*
* This function is private to the EAL.
*/
unsigned eal_cpu_core_id(unsigned lcore_id);
/**
* Check if cpu is present.
*
* This function is private to the EAL.
*/
int eal_cpu_detected(unsigned lcore_id);
/**
* Set TSC frequency from precise value or estimation
*
* This function is private to the EAL.
*/
void set_tsc_freq(void);
/**
* Get precise TSC frequency from system
*
* This function is private to the EAL.
*/
uint64_t get_tsc_freq(void);
/**
* Get TSC frequency if the architecture supports.
*
* This function is private to the EAL.
*
* @return
* The number of TSC cycles in one second.
* Returns zero if the architecture support is not available.
*/
uint64_t get_tsc_freq_arch(void);
/**
* Allocate a free lcore to associate to a non-EAL thread.
*
* @return
* - the id of a lcore with role ROLE_NON_EAL on success.
* - RTE_MAX_LCORE if none was available or initializing was refused (see
* rte_lcore_callback_register).
*/
unsigned int eal_lcore_non_eal_allocate(void);
/**
* Release the lcore used by a non-EAL thread.
* Counterpart of eal_lcore_non_eal_allocate().
*
* @param lcore_id
* The lcore with role ROLE_NON_EAL to release.
*/
void eal_lcore_non_eal_release(unsigned int lcore_id);
/**
* Prepare physical memory mapping
* i.e. hugepages on Linux and
* contigmem on BSD.
*
* This function is private to the EAL.
*/
int rte_eal_hugepage_init(void);
/**
* Creates memory mapping in secondary process
* i.e. hugepages on Linux and
* contigmem on BSD.
*
* This function is private to the EAL.
*/
int rte_eal_hugepage_attach(void);
/**
* Detaches all memory mappings from a process.
*
* This function is private to the EAL.
*/
int rte_eal_memory_detach(void);
/**
* Find a bus capable of identifying a device.
*
* @param str
* A device identifier (PCI address, virtual PMD name, ...).
*
* @return
* A valid bus handle if found.
* NULL if no bus is able to parse this device.
*/
struct rte_bus *rte_bus_find_by_device_name(const char *str);
/**
* For each device on the buses, call the driver-specific function for
* device cleanup.
*
* @return
* 0 for successful cleanup
* !0 otherwise
*/
int eal_bus_cleanup(void);
/**
* Create the unix channel for primary/secondary communication.
*
* @return
* 0 on success;
* (<0) on failure.
*/
int rte_mp_channel_init(void);
/**
* Primary/secondary communication cleanup.
*/
void rte_mp_channel_cleanup(void);
/**
* @internal
* Parse a device string and store its information in an
* rte_devargs structure.
*
* A device description is split by layers of abstraction of the device:
* bus, class and driver. Each layer will offer a set of properties that
* can be applied either to configure or recognize a device.
*
* This function will parse those properties and prepare the rte_devargs
* to be given to each layers for processing.
*
* Note: if the "data" field of the devargs points to devstr,
* then no dynamic allocation is performed and the rte_devargs
* can be safely discarded.
*
* Otherwise ``data`` will hold a workable copy of devstr, that will be
* used by layers descriptors within rte_devargs. In this case,
* any rte_devargs should be cleaned-up before being freed.
*
* @param da
* rte_devargs structure to fill.
*
* @param devstr
* Device string.
*
* @return
* 0 on success.
* Negative errno values on error (rte_errno is set).
*/
int
rte_devargs_layers_parse(struct rte_devargs *devargs,
const char *devstr);
/*
* probe a device at local process.
*
* @param devargs
* Device arguments including bus, class and driver properties.
* @param new_dev
* new device be probed as output.
* @return
* 0 on success, negative on error.
*/
int local_dev_probe(const char *devargs, struct rte_device **new_dev);
/**
* Hotplug remove a given device from a specific bus at local process.
*
* @param dev
* Data structure of the device to remove.
* @return
* 0 on success, negative on error.
*/
int local_dev_remove(struct rte_device *dev);
/**
* Iterate over all buses to find the corresponding bus to handle the sigbus
* error.
* @param failure_addr
* Pointer of the fault address of the sigbus error.
*
* @return
* 0 success to handle the sigbus.
* -1 failed to handle the sigbus
* 1 no bus can handler the sigbus
*/
int rte_bus_sigbus_handler(const void *failure_addr);
/**
* @internal
* Register the sigbus handler.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
int
dev_sigbus_handler_register(void);
/**
* @internal
* Unregister the sigbus handler.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
int
dev_sigbus_handler_unregister(void);
/**
* Get OS-specific EAL mapping base address.
*/
uint64_t
eal_get_baseaddr(void);
void *
eal_malloc_no_trace(const char *type, size_t size, unsigned int align);
void eal_free_no_trace(void *addr);
/** Options for eal_file_open(). */
enum eal_open_flags {
/** Open file for reading. */
EAL_OPEN_READONLY = 0x00,
/** Open file for reading and writing. */
EAL_OPEN_READWRITE = 0x02,
/**
* Create the file if it doesn't exist.
* New files are only accessible to the owner (0600 equivalent).
*/
EAL_OPEN_CREATE = 0x04
};
/**
* Open or create a file.
*
* @param path
* Path to the file.
* @param flags
* A combination of eal_open_flags controlling operation and FD behavior.
* @return
* Open file descriptor on success, (-1) on failure and rte_errno is set.
*/
int
eal_file_open(const char *path, int flags);
/** File locking operation. */
enum eal_flock_op {
EAL_FLOCK_SHARED, /**< Acquire a shared lock. */
EAL_FLOCK_EXCLUSIVE, /**< Acquire an exclusive lock. */
EAL_FLOCK_UNLOCK /**< Release a previously taken lock. */
};
/** Behavior on file locking conflict. */
enum eal_flock_mode {
EAL_FLOCK_WAIT, /**< Wait until the file gets unlocked to lock it. */
EAL_FLOCK_RETURN /**< Return immediately if the file is locked. */
};
/**
* Lock or unlock the file.
*
* On failure @code rte_errno @endcode is set to the error code
* specified by POSIX flock(3) description.
*
* @param fd
* Opened file descriptor.
* @param op
* Operation to perform.
* @param mode
* Behavior on conflict.
* @return
* 0 on success, (-1) on failure.
*/
int
eal_file_lock(int fd, enum eal_flock_op op, enum eal_flock_mode mode);
/**
* Truncate or extend the file to the specified size.
*
* On failure @code rte_errno @endcode is set to the error code
* specified by POSIX ftruncate(3) description.
*
* @param fd
* Opened file descriptor.
* @param size
* Desired file size.
* @return
* 0 on success, (-1) on failure.
*/
int
eal_file_truncate(int fd, ssize_t size);
/**
* Reserve a region of virtual memory.
*
* Use eal_mem_free() to free reserved memory.
*
* @param requested_addr
* A desired reservation address which must be page-aligned.
* The system might not respect it.
* NULL means the address will be chosen by the system.
* @param size
* Reservation size. Must be a multiple of system page size.
* @param flags
* Reservation options, a combination of eal_mem_reserve_flags.
* @returns
* Starting address of the reserved area on success, NULL on failure.
* Callers must not access this memory until remapping it.
*/
void *
eal_mem_reserve(void *requested_addr, size_t size, int flags);
/**
* Free memory obtained by eal_mem_reserve() and possibly allocated.
*
* If *virt* and *size* describe a part of the reserved region,
* only this part of the region is freed (accurately up to the system
* page size). If *virt* points to allocated memory, *size* must match
* the one specified on allocation. The behavior is undefined
* if the memory pointed by *virt* is obtained from another source
* than listed above.
*
* @param virt
* A virtual address in a region previously reserved.
* @param size
* Number of bytes to unreserve.
*/
void
eal_mem_free(void *virt, size_t size);
/**
* Configure memory region inclusion into dumps.
*
* @param virt
* Starting address of the region.
* @param size
* Size of the region.
* @param dump
* True to include memory into dumps, false to exclude.
* @return
* 0 on success, (-1) on failure and rte_errno is set.
*/
int
eal_mem_set_dump(void *virt, size_t size, bool dump);
/**
* Sets the runtime directory of DPDK
*
* @param run_dir
* The new runtime directory path of DPDK
* @return
* 0 on success, (-1) on failure.
*/
int
eal_set_runtime_dir(const char *run_dir);
/**
* Get the internal configuration structure.
*
* @return
* A pointer to the internal configuration structure.
*/
struct internal_config *
eal_get_internal_configuration(void);
/**
* Get the current value of the rte_application_usage pointer
*
* @return
* Pointer to the current value of rte_application_usage .
*/
rte_usage_hook_t
eal_get_application_usage_hook(void);
/**
* Instruct primary process that a secondary process wants to attach.
*/
bool __rte_mp_enable(void);
/**
* Init per-lcore info in current thread.
*
* @param lcore_id
* identifier of lcore.
* @param cpuset
* CPU affinity for this thread.
*/
void __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset);
/**
* Uninitialize per-lcore info for current thread.
*/
void __rte_thread_uninit(void);
/**
* asprintf(3) replacement for Windows.
*/
#ifdef RTE_EXEC_ENV_WINDOWS
__rte_format_printf(2, 3)
int eal_asprintf(char **buffer, const char *format, ...);
#define asprintf(buffer, format, ...) \
eal_asprintf(buffer, format, ##__VA_ARGS__)
#endif
#endif /* _EAL_PRIVATE_H_ */

91
Programs/hugePage/common/eal_thread.h Normal file
View File

@@ -0,0 +1,91 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef EAL_THREAD_H
#define EAL_THREAD_H
#include "rte_common.h"
#include "rte_lcore.h"
/**
* Basic loop of EAL thread, called for each worker thread by rte_eal_init().
*
* @param arg
* The lcore_id (passed as an integer) of this worker thread.
*/
__rte_noreturn void *eal_thread_loop(void *arg);
/**
* Get the NUMA socket id from cpu id.
* This function is private to EAL.
*
* @param cpu_id
* The logical process id.
* @return
* socket_id or SOCKET_ID_ANY
*/
unsigned eal_cpu_socket_id(unsigned cpu_id);
/**
* Default buffer size to use with eal_thread_dump_affinity()
*/
#define RTE_CPU_AFFINITY_STR_LEN 256
/**
* Dump the cpuset as a human readable string.
* This function is private to EAL.
*
* Note:
* If the dump size is greater than the size of given buffer,
* the string will be truncated and with '\0' at the end.
*
* @param cpuset
* The CPU affinity object to dump.
* @param str
* The string buffer the cpuset will dump to.
* @param size
* The string buffer size.
* @return
* 0 for success, -1 if truncation happens.
*/
int
eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size);
/**
* Dump the current thread cpuset.
* This is a wrapper on eal_thread_dump_affinity().
*/
int
eal_thread_dump_current_affinity(char *str, unsigned int size);
/**
* Called by the main thread to wake up a worker in 'WAIT' state.
* This function blocks until the worker acknowledge it started processing a
* new command.
* This function is private to EAL.
*
* @param worker_id
* The lcore_id of a worker thread.
* @return
* 0 on success, negative errno on error
*/
int
eal_thread_wake_worker(unsigned int worker_id);
/**
* Called by a worker thread to sleep after entering 'WAIT' state.
* This function is private to EAL.
*/
void
eal_thread_wait_command(void);
/**
* Called by a worker thread to acknowledge new command after leaving 'WAIT'
* state.
* This function is private to EAL.
*/
void
eal_thread_ack_command(void);
#endif /* EAL_THREAD_H */

116
Programs/hugePage/common/eal_trace.h Normal file
View File

@@ -0,0 +1,116 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2020 Marvell International Ltd.
*/
#ifndef __EAL_TRACE_H
#define __EAL_TRACE_H
#include "rte_cycles.h"
#include "rte_log.h"
#include "rte_malloc.h"
#include "rte_spinlock.h"
#include "rte_trace.h"
#include "rte_trace_point.h"
#include "rte_uuid.h"
#include "eal_private.h"
#include "eal_thread.h"
#define trace_err(fmt, args...) \
RTE_LOG(ERR, EAL, "%s():%u " fmt "\n", __func__, __LINE__, ## args)
#define trace_crit(fmt, args...) \
RTE_LOG(CRIT, EAL, "%s():%u " fmt "\n", __func__, __LINE__, ## args)
#define TRACE_CTF_MAGIC 0xC1FC1FC1
#define TRACE_MAX_ARGS 32
struct trace_point {
STAILQ_ENTRY(trace_point) next;
rte_trace_point_t *handle;
const char *name;
char *ctf_field;
};
enum trace_area_e {
TRACE_AREA_HEAP,
TRACE_AREA_HUGEPAGE,
};
struct thread_mem_meta {
void *mem;
enum trace_area_e area;
};
struct trace_arg {
STAILQ_ENTRY(trace_arg) next;
char *val;
};
struct trace {
char *dir;
int register_errno;
uint32_t status;
enum rte_trace_mode mode;
rte_uuid_t uuid;
uint32_t buff_len;
STAILQ_HEAD(, trace_arg) args;
uint32_t nb_trace_points;
uint32_t nb_trace_mem_list;
struct thread_mem_meta *lcore_meta;
uint64_t epoch_sec;
uint64_t epoch_nsec;
uint64_t uptime_ticks;
char *ctf_meta;
uint32_t ctf_meta_offset_freq;
uint32_t ctf_meta_offset_freq_off_s;
uint32_t ctf_meta_offset_freq_off;
uint16_t ctf_fixup_done;
rte_spinlock_t lock;
};
/* Helper functions */
static inline uint16_t
trace_id_get(rte_trace_point_t *trace)
{
return (*trace & __RTE_TRACE_FIELD_ID_MASK) >>
__RTE_TRACE_FIELD_ID_SHIFT;
}
static inline size_t
trace_mem_sz(uint32_t len)
{
return len + sizeof(struct __rte_trace_header);
}
/* Trace object functions */
struct trace *trace_obj_get(void);
/* Trace point list functions */
STAILQ_HEAD(trace_point_head, trace_point);
struct trace_point_head *trace_list_head_get(void);
/* Util functions */
const char *trace_mode_to_string(enum rte_trace_mode mode);
const char *trace_area_to_string(enum trace_area_e area);
int trace_args_apply(const char *arg);
void trace_bufsz_args_apply(void);
bool trace_has_duplicate_entry(void);
void trace_uuid_generate(void);
int trace_metadata_create(void);
void trace_metadata_destroy(void);
char *trace_metadata_fixup_field(const char *field);
int trace_epoch_time_save(void);
void trace_mem_free(void);
void trace_mem_per_thread_free(void);
/* EAL interface */
int eal_trace_init(void);
void eal_trace_fini(void);
int eal_trace_args_save(const char *val);
void eal_trace_args_free(void);
int eal_trace_dir_args_save(const char *val);
int eal_trace_mode_args_save(const char *val);
int eal_trace_bufsz_args_save(const char *val);
#endif /* __EAL_TRACE_H */

1117
Programs/hugePage/common/generic/rte_atomic.h Normal file
View File

File diff suppressed because it is too large Load Diff

247
Programs/hugePage/common/generic/rte_byteorder.h Normal file
View File

@@ -0,0 +1,247 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_BYTEORDER_H_
#define _RTE_BYTEORDER_H_
/**
* @file
*
* Byte Swap Operations
*
* This file defines a generic API for byte swap operations. Part of
* the implementation is architecture-specific.
*/
#include <stdint.h>
#ifdef RTE_EXEC_ENV_FREEBSD
#include <sys/endian.h>
#elif defined RTE_EXEC_ENV_LINUX
#include <endian.h>
#endif
#include "rte_common.h"
#include "rte_config.h"
/*
* Compile-time endianness detection
*/
#define RTE_BIG_ENDIAN 1
#define RTE_LITTLE_ENDIAN 2
#if defined __BYTE_ORDER__
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define RTE_BYTE_ORDER RTE_BIG_ENDIAN
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define RTE_BYTE_ORDER RTE_LITTLE_ENDIAN
#endif /* __BYTE_ORDER__ */
#elif defined __BYTE_ORDER
#if __BYTE_ORDER == __BIG_ENDIAN
#define RTE_BYTE_ORDER RTE_BIG_ENDIAN
#elif __BYTE_ORDER == __LITTLE_ENDIAN
#define RTE_BYTE_ORDER RTE_LITTLE_ENDIAN
#endif /* __BYTE_ORDER */
#elif defined __BIG_ENDIAN__
#define RTE_BYTE_ORDER RTE_BIG_ENDIAN
#elif defined __LITTLE_ENDIAN__
#define RTE_BYTE_ORDER RTE_LITTLE_ENDIAN
#endif
#if !defined(RTE_BYTE_ORDER)
#error Unknown endianness.
#endif
#define RTE_STATIC_BSWAP16(v) \
((((uint16_t)(v) & UINT16_C(0x00ff)) << 8) | \
(((uint16_t)(v) & UINT16_C(0xff00)) >> 8))
#define RTE_STATIC_BSWAP32(v) \
((((uint32_t)(v) & UINT32_C(0x000000ff)) << 24) | \
(((uint32_t)(v) & UINT32_C(0x0000ff00)) << 8) | \
(((uint32_t)(v) & UINT32_C(0x00ff0000)) >> 8) | \
(((uint32_t)(v) & UINT32_C(0xff000000)) >> 24))
#define RTE_STATIC_BSWAP64(v) \
((((uint64_t)(v) & UINT64_C(0x00000000000000ff)) << 56) | \
(((uint64_t)(v) & UINT64_C(0x000000000000ff00)) << 40) | \
(((uint64_t)(v) & UINT64_C(0x0000000000ff0000)) << 24) | \
(((uint64_t)(v) & UINT64_C(0x00000000ff000000)) << 8) | \
(((uint64_t)(v) & UINT64_C(0x000000ff00000000)) >> 8) | \
(((uint64_t)(v) & UINT64_C(0x0000ff0000000000)) >> 24) | \
(((uint64_t)(v) & UINT64_C(0x00ff000000000000)) >> 40) | \
(((uint64_t)(v) & UINT64_C(0xff00000000000000)) >> 56))
/*
* These macros are functionally similar to rte_cpu_to_(be|le)(16|32|64)(),
* they take values in host CPU order and return them converted to the
* intended endianness.
*
* They resolve at compilation time to integer constants which can safely be
* used with static initializers, since those cannot involve function calls.
*
* On the other hand, they are not as optimized as their rte_cpu_to_*()
* counterparts, therefore applications should refrain from using them on
* variable values, particularly inside performance-sensitive code.
*/
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
#define RTE_BE16(v) (rte_be16_t)(v)
#define RTE_BE32(v) (rte_be32_t)(v)
#define RTE_BE64(v) (rte_be64_t)(v)
#define RTE_LE16(v) (rte_le16_t)(RTE_STATIC_BSWAP16(v))
#define RTE_LE32(v) (rte_le32_t)(RTE_STATIC_BSWAP32(v))
#define RTE_LE64(v) (rte_le64_t)(RTE_STATIC_BSWAP64(v))
#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
#define RTE_BE16(v) (rte_be16_t)(RTE_STATIC_BSWAP16(v))
#define RTE_BE32(v) (rte_be32_t)(RTE_STATIC_BSWAP32(v))
#define RTE_BE64(v) (rte_be64_t)(RTE_STATIC_BSWAP64(v))
#define RTE_LE16(v) (rte_le16_t)(v)
#define RTE_LE32(v) (rte_le32_t)(v)
#define RTE_LE64(v) (rte_le64_t)(v)
#else
#error Unsupported endianness.
#endif
/*
* The following types should be used when handling values according to a
* specific byte ordering, which may differ from that of the host CPU.
*
* Libraries, public APIs and applications are encouraged to use them for
* documentation purposes.
*/
typedef uint16_t rte_be16_t; /**< 16-bit big-endian value. */
typedef uint32_t rte_be32_t; /**< 32-bit big-endian value. */
typedef uint64_t rte_be64_t; /**< 64-bit big-endian value. */
typedef uint16_t rte_le16_t; /**< 16-bit little-endian value. */
typedef uint32_t rte_le32_t; /**< 32-bit little-endian value. */
typedef uint64_t rte_le64_t; /**< 64-bit little-endian value. */
/*
* An internal function to swap bytes in a 16-bit value.
*
* It is used by rte_bswap16() when the value is constant. Do not use
* this function directly; rte_bswap16() is preferred.
*/
static inline uint16_t
rte_constant_bswap16(uint16_t x)
{
return (uint16_t)RTE_STATIC_BSWAP16(x);
}
/*
* An internal function to swap bytes in a 32-bit value.
*
* It is used by rte_bswap32() when the value is constant. Do not use
* this function directly; rte_bswap32() is preferred.
*/
static inline uint32_t
rte_constant_bswap32(uint32_t x)
{
return (uint32_t)RTE_STATIC_BSWAP32(x);
}
/*
* An internal function to swap bytes of a 64-bit value.
*
* It is used by rte_bswap64() when the value is constant. Do not use
* this function directly; rte_bswap64() is preferred.
*/
static inline uint64_t
rte_constant_bswap64(uint64_t x)
{
return (uint64_t)RTE_STATIC_BSWAP64(x);
}
#ifdef __DOXYGEN__
/**
* Swap bytes in a 16-bit value.
*/
static uint16_t rte_bswap16(uint16_t _x);
/**
* Swap bytes in a 32-bit value.
*/
static uint32_t rte_bswap32(uint32_t x);
/**
* Swap bytes in a 64-bit value.
*/
static uint64_t rte_bswap64(uint64_t x);
/**
* Convert a 16-bit value from CPU order to little endian.
*/
static rte_le16_t rte_cpu_to_le_16(uint16_t x);
/**
* Convert a 32-bit value from CPU order to little endian.
*/
static rte_le32_t rte_cpu_to_le_32(uint32_t x);
/**
* Convert a 64-bit value from CPU order to little endian.
*/
static rte_le64_t rte_cpu_to_le_64(uint64_t x);
/**
* Convert a 16-bit value from CPU order to big endian.
*/
static rte_be16_t rte_cpu_to_be_16(uint16_t x);
/**
* Convert a 32-bit value from CPU order to big endian.
*/
static rte_be32_t rte_cpu_to_be_32(uint32_t x);
/**
* Convert a 64-bit value from CPU order to big endian.
*/
static rte_be64_t rte_cpu_to_be_64(uint64_t x);
/**
* Convert a 16-bit value from little endian to CPU order.
*/
static uint16_t rte_le_to_cpu_16(rte_le16_t x);
/**
* Convert a 32-bit value from little endian to CPU order.
*/
static uint32_t rte_le_to_cpu_32(rte_le32_t x);
/**
* Convert a 64-bit value from little endian to CPU order.
*/
static uint64_t rte_le_to_cpu_64(rte_le64_t x);
/**
* Convert a 16-bit value from big endian to CPU order.
*/
static uint16_t rte_be_to_cpu_16(rte_be16_t x);
/**
* Convert a 32-bit value from big endian to CPU order.
*/
static uint32_t rte_be_to_cpu_32(rte_be32_t x);
/**
* Convert a 64-bit value from big endian to CPU order.
*/
static uint64_t rte_be_to_cpu_64(rte_be64_t x);
#endif /* __DOXYGEN__ */
#ifdef RTE_FORCE_INTRINSICS
#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)
#define rte_bswap16(x) __builtin_bswap16(x)
#endif
#define rte_bswap32(x) __builtin_bswap32(x)
#define rte_bswap64(x) __builtin_bswap64(x)
#endif
#endif /* _RTE_BYTEORDER_H_ */

107
Programs/hugePage/common/generic/rte_cpuflags.h Normal file
View File

@@ -0,0 +1,107 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_CPUFLAGS_H_
#define _RTE_CPUFLAGS_H_
/**
* @file
* Architecture specific API to determine available CPU features at runtime.
*/
#include "rte_common.h"
#include <errno.h>
#include "rte_compat.h"
/**
* Structure used to describe platform-specific intrinsics that may or may not
* be supported at runtime.
*/
struct rte_cpu_intrinsics {
uint32_t power_monitor : 1;
/**< indicates support for rte_power_monitor function */
uint32_t power_pause : 1;
/**< indicates support for rte_power_pause function */
uint32_t power_monitor_multi : 1;
/**< indicates support for rte_power_monitor_multi function */
};
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Check CPU support for various intrinsics at runtime.
*
* @param intrinsics
* Pointer to a structure to be filled.
*/
__rte_experimental
void
rte_cpu_get_intrinsics_support(struct rte_cpu_intrinsics *intrinsics);
/**
* Enumeration of all CPU features supported
*/
__extension__
enum rte_cpu_flag_t;
/**
* Get name of CPU flag
*
* @param feature
* CPU flag ID
* @return
* flag name
* NULL if flag ID is invalid
*/
__extension__
const char *
rte_cpu_get_flag_name(enum rte_cpu_flag_t feature);
/**
* Function for checking a CPU flag availability
*
* @param feature
* CPU flag to query CPU for
* @return
* 1 if flag is available
* 0 if flag is not available
* -ENOENT if flag is invalid
*/
__extension__
int
rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature);
/**
* This function checks that the currently used CPU supports the CPU features
* that were specified at compile time. It is called automatically within the
* EAL, so does not need to be used by applications. This version returns a
* result so that decisions may be made (for instance, graceful shutdowns).
*/
int
rte_cpu_is_supported(void);
/**
* This function attempts to retrieve a value from the auxiliary vector.
* If it is unsuccessful, the result will be 0, and errno will be set.
*
* @return A value from the auxiliary vector. When the value is 0, check
* errno to determine if an error occurred.
*/
unsigned long
rte_cpu_getauxval(unsigned long type);
/**
* This function retrieves a value from the auxiliary vector, and compares it
* as a string against the value retrieved.
*
* @return The result of calling strcmp() against the value retrieved from
* the auxiliary vector. When the value is 0 (meaning a match is found),
* check errno to determine if an error occurred.
*/
int
rte_cpu_strcmp_auxval(unsigned long type, const char *str);
#endif /* _RTE_CPUFLAGS_H_ */

178
Programs/hugePage/common/generic/rte_cycles.h Normal file
View File

@@ -0,0 +1,178 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation.
* Copyright(c) 2013 6WIND S.A.
*/
#ifndef _RTE_CYCLES_H_
#define _RTE_CYCLES_H_
/**
* @file
*
* Simple Time Reference Functions (Cycles and HPET).
*/
#include <stdint.h>
#include "rte_debug.h"
#include "rte_atomic.h"
#define MS_PER_S 1000
#define US_PER_S 1000000
#define NS_PER_S 1000000000
enum timer_source {
EAL_TIMER_TSC = 0,
EAL_TIMER_HPET
};
extern enum timer_source eal_timer_source;
/**
* Get the measured frequency of the RDTSC counter
*
* @return
* The TSC frequency for this lcore
*/
uint64_t
rte_get_tsc_hz(void);
/**
* Return the number of TSC cycles since boot
*
* @return
* the number of cycles
*/
static inline uint64_t
rte_get_tsc_cycles(void);
#ifdef RTE_LIBEAL_USE_HPET
/**
* Return the number of HPET cycles since boot
*
* This counter is global for all execution units. The number of
* cycles in one second can be retrieved using rte_get_hpet_hz().
*
* @return
* the number of cycles
*/
uint64_t
rte_get_hpet_cycles(void);
/**
* Get the number of HPET cycles in one second.
*
* @return
* The number of cycles in one second.
*/
uint64_t
rte_get_hpet_hz(void);
/**
* Initialise the HPET for use. This must be called before the rte_get_hpet_hz
* and rte_get_hpet_cycles APIs are called. If this function does not succeed,
* then the HPET functions are unavailable and should not be called.
*
* @param make_default
* If set, the hpet timer becomes the default timer whose values are
* returned by the rte_get_timer_hz/cycles API calls
*
* @return
* 0 on success,
* -1 on error, and the make_default parameter is ignored.
*/
int rte_eal_hpet_init(int make_default);
#endif
/**
* Get the number of cycles since boot from the default timer.
*
* @return
* The number of cycles
*/
static inline uint64_t
rte_get_timer_cycles(void)
{
#ifdef RTE_LIBEAL_USE_HPET
switch(eal_timer_source) {
case EAL_TIMER_TSC:
#endif
return rte_get_tsc_cycles();
#ifdef RTE_LIBEAL_USE_HPET
case EAL_TIMER_HPET:
return rte_get_hpet_cycles();
default: rte_panic("Invalid timer source specified\n");
}
#endif
}
/**
* Get the number of cycles in one second for the default timer.
*
* @return
* The number of cycles in one second.
*/
static inline uint64_t
rte_get_timer_hz(void)
{
#ifdef RTE_LIBEAL_USE_HPET
switch(eal_timer_source) {
case EAL_TIMER_TSC:
#endif
return rte_get_tsc_hz();
#ifdef RTE_LIBEAL_USE_HPET
case EAL_TIMER_HPET:
return rte_get_hpet_hz();
default: rte_panic("Invalid timer source specified\n");
}
#endif
}
/**
* Wait at least us microseconds.
* This function can be replaced with user-defined function.
* @see rte_delay_us_callback_register
*
* @param us
* The number of microseconds to wait.
*/
extern void
(*rte_delay_us)(unsigned int us);
/**
* Wait at least ms milliseconds.
*
* @param ms
* The number of milliseconds to wait.
*/
static inline void
rte_delay_ms(unsigned ms)
{
rte_delay_us(ms * 1000);
}
/**
* Blocking delay function.
*
* @param us
* Number of microseconds to wait.
*/
void rte_delay_us_block(unsigned int us);
/**
* Delay function that uses system sleep.
* Does not block the CPU core.
*
* @param us
* Number of microseconds to wait.
*/
void rte_delay_us_sleep(unsigned int us);
/**
* Replace rte_delay_us with user defined function.
*
* @param userfunc
* User function which replaces rte_delay_us. rte_delay_us_block restores
* builtin block delay function.
*/
void rte_delay_us_callback_register(void(*userfunc)(unsigned int));
#endif /* _RTE_CYCLES_H_ */

399
Programs/hugePage/common/generic/rte_io.h Normal file
View File

@@ -0,0 +1,399 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2016 Cavium, Inc
*/
#ifndef _RTE_IO_H_
#define _RTE_IO_H_
/**
* @file
* I/O device memory operations
*
* This file defines the generic API for I/O device memory read/write operations
*/
#include <stdint.h>
#include "rte_common.h"
#include "rte_compat.h"
#include "rte_atomic.h"
#ifdef __DOXYGEN__
/**
* Read a 8-bit value from I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint8_t
rte_read8_relaxed(const volatile void *addr);
/**
* Read a 16-bit value from I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint16_t
rte_read16_relaxed(const volatile void *addr);
/**
* Read a 32-bit value from I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint32_t
rte_read32_relaxed(const volatile void *addr);
/**
* Read a 64-bit value from I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint64_t
rte_read64_relaxed(const volatile void *addr);
/**
* Write a 8-bit value to I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write8_relaxed(uint8_t value, volatile void *addr);
/**
* Write a 16-bit value to I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write16_relaxed(uint16_t value, volatile void *addr);
/**
* Write a 32-bit value to I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write32_relaxed(uint32_t value, volatile void *addr);
/**
* Write a 64-bit value to I/O device memory address *addr*.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write64_relaxed(uint64_t value, volatile void *addr);
/**
* Read a 8-bit value from I/O device memory address *addr*.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint8_t
rte_read8(const volatile void *addr);
/**
* Read a 16-bit value from I/O device memory address *addr*.
*
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint16_t
rte_read16(const volatile void *addr);
/**
* Read a 32-bit value from I/O device memory address *addr*.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint32_t
rte_read32(const volatile void *addr);
/**
* Read a 64-bit value from I/O device memory address *addr*.
*
* @param addr
* I/O memory address to read the value from
* @return
* read value
*/
static inline uint64_t
rte_read64(const volatile void *addr);
/**
* Write a 8-bit value to I/O device memory address *addr*.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write8(uint8_t value, volatile void *addr);
/**
* Write a 16-bit value to I/O device memory address *addr*.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write16(uint16_t value, volatile void *addr);
/**
* Write a 32-bit value to I/O device memory address *addr*.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write32(uint32_t value, volatile void *addr);
/**
* Write a 64-bit value to I/O device memory address *addr*.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
static inline void
rte_write64(uint64_t value, volatile void *addr);
/**
* Write a 32-bit value to I/O device memory address addr using write
* combining memory write protocol. Depending on the platform write combining
* may not be available and/or may be treated as a hint and the behavior may
* fallback to a regular store.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
__rte_experimental
static inline void
rte_write32_wc(uint32_t value, volatile void *addr);
/**
* Write a 32-bit value to I/O device memory address addr using write
* combining memory write protocol. Depending on the platform write combining
* may not be available and/or may be treated as a hint and the behavior may
* fallback to a regular store.
*
* The relaxed version does not have additional I/O memory barrier, useful in
* accessing the device registers of integrated controllers which implicitly
* strongly ordered with respect to memory access.
*
* @param value
* Value to write
* @param addr
* I/O memory address to write the value to
*/
__rte_experimental
static inline void
rte_write32_wc_relaxed(uint32_t value, volatile void *addr);
#endif /* __DOXYGEN__ */
#ifndef RTE_OVERRIDE_IO_H
static __rte_always_inline uint8_t
rte_read8_relaxed(const volatile void *addr)
{
return *(const volatile uint8_t *)addr;
}
static __rte_always_inline uint16_t
rte_read16_relaxed(const volatile void *addr)
{
return *(const volatile uint16_t *)addr;
}
static __rte_always_inline uint32_t
rte_read32_relaxed(const volatile void *addr)
{
return *(const volatile uint32_t *)addr;
}
static __rte_always_inline uint64_t
rte_read64_relaxed(const volatile void *addr)
{
return *(const volatile uint64_t *)addr;
}
static __rte_always_inline void
rte_write8_relaxed(uint8_t value, volatile void *addr)
{
*(volatile uint8_t *)addr = value;
}
static __rte_always_inline void
rte_write16_relaxed(uint16_t value, volatile void *addr)
{
*(volatile uint16_t *)addr = value;
}
static __rte_always_inline void
rte_write32_relaxed(uint32_t value, volatile void *addr)
{
*(volatile uint32_t *)addr = value;
}
static __rte_always_inline void
rte_write64_relaxed(uint64_t value, volatile void *addr)
{
*(volatile uint64_t *)addr = value;
}
static __rte_always_inline uint8_t
rte_read8(const volatile void *addr)
{
uint8_t val;
val = rte_read8_relaxed(addr);
rte_io_rmb();
return val;
}
static __rte_always_inline uint16_t
rte_read16(const volatile void *addr)
{
uint16_t val;
val = rte_read16_relaxed(addr);
rte_io_rmb();
return val;
}
static __rte_always_inline uint32_t
rte_read32(const volatile void *addr)
{
uint32_t val;
val = rte_read32_relaxed(addr);
rte_io_rmb();
return val;
}
static __rte_always_inline uint64_t
rte_read64(const volatile void *addr)
{
uint64_t val;
val = rte_read64_relaxed(addr);
rte_io_rmb();
return val;
}
static __rte_always_inline void
rte_write8(uint8_t value, volatile void *addr)
{
rte_io_wmb();
rte_write8_relaxed(value, addr);
}
static __rte_always_inline void
rte_write16(uint16_t value, volatile void *addr)
{
rte_io_wmb();
rte_write16_relaxed(value, addr);
}
static __rte_always_inline void
rte_write32(uint32_t value, volatile void *addr)
{
rte_io_wmb();
rte_write32_relaxed(value, addr);
}
static __rte_always_inline void
rte_write64(uint64_t value, volatile void *addr)
{
rte_io_wmb();
rte_write64_relaxed(value, addr);
}
#ifndef RTE_NATIVE_WRITE32_WC
static __rte_always_inline void
rte_write32_wc(uint32_t value, volatile void *addr)
{
rte_write32(value, addr);
}
static __rte_always_inline void
rte_write32_wc_relaxed(uint32_t value, volatile void *addr)
{
rte_write32_relaxed(value, addr);
}
#endif /* RTE_NATIVE_WRITE32_WC */
#endif /* RTE_OVERRIDE_IO_H */
#endif /* _RTE_IO_H_ */

116
Programs/hugePage/common/generic/rte_memcpy.h Normal file
View File

@@ -0,0 +1,116 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_MEMCPY_H_
#define _RTE_MEMCPY_H_
/**
* @file
*
* Functions for vectorised implementation of memcpy().
*/
/**
* Copy 16 bytes from one location to another using optimised
* instructions. The locations should not overlap.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
*/
static inline void
rte_mov16(uint8_t *dst, const uint8_t *src);
/**
* Copy 32 bytes from one location to another using optimised
* instructions. The locations should not overlap.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
*/
static inline void
rte_mov32(uint8_t *dst, const uint8_t *src);
#ifdef __DOXYGEN__
/**
* Copy 48 bytes from one location to another using optimised
* instructions. The locations should not overlap.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
*/
static inline void
rte_mov48(uint8_t *dst, const uint8_t *src);
#endif /* __DOXYGEN__ */
/**
* Copy 64 bytes from one location to another using optimised
* instructions. The locations should not overlap.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
*/
static inline void
rte_mov64(uint8_t *dst, const uint8_t *src);
/**
* Copy 128 bytes from one location to another using optimised
* instructions. The locations should not overlap.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
*/
static inline void
rte_mov128(uint8_t *dst, const uint8_t *src);
/**
* Copy 256 bytes from one location to another using optimised
* instructions. The locations should not overlap.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
*/
static inline void
rte_mov256(uint8_t *dst, const uint8_t *src);
#ifdef __DOXYGEN__
/**
* Copy bytes from one location to another. The locations must not overlap.
*
* @note This is implemented as a macro, so it's address should not be taken
* and care is needed as parameter expressions may be evaluated multiple times.
*
* @note For x86 platforms to enable the AVX-512 memcpy implementation, set
* -DRTE_MEMCPY_AVX512 macro in CFLAGS, or define the RTE_MEMCPY_AVX512 macro
* explicitly in the source file before including the rte_memcpy header file.
*
* @param dst
* Pointer to the destination of the data.
* @param src
* Pointer to the source data.
* @param n
* Number of bytes to copy.
* @return
* Pointer to the destination data.
*/
static void *
rte_memcpy(void *dst, const void *src, size_t n);
#endif /* __DOXYGEN__ */
#endif /* _RTE_MEMCPY_H_ */

143
Programs/hugePage/common/generic/rte_pause.h Normal file
View File

@@ -0,0 +1,143 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017 Cavium, Inc
* Copyright(c) 2019 Arm Limited
*/
#ifndef _RTE_PAUSE_H_
#define _RTE_PAUSE_H_
/**
* @file
*
* CPU pause operation.
*
*/
#include <stdint.h>
#include <assert.h>
#include "../rte_common.h"
#include "rte_atomic.h"
/**
* Pause CPU execution for a short while
*
* This call is intended for tight loops which poll a shared resource or wait
* for an event. A short pause within the loop may reduce the power consumption.
*/
static inline void rte_pause(void);
/**
* Wait for *addr to be updated with a 16-bit expected value, with a relaxed
* memory ordering model meaning the loads around this API can be reordered.
*
* @param addr
* A pointer to the memory location.
* @param expected
* A 16-bit expected value to be in the memory location.
* @param memorder
* Two different memory orders that can be specified:
* __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
* C++11 memory orders with the same names, see the C++11 standard or
* the GCC wiki on atomic synchronization for detailed definition.
*/
static __rte_always_inline void
rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
int memorder);
/**
* Wait for *addr to be updated with a 32-bit expected value, with a relaxed
* memory ordering model meaning the loads around this API can be reordered.
*
* @param addr
* A pointer to the memory location.
* @param expected
* A 32-bit expected value to be in the memory location.
* @param memorder
* Two different memory orders that can be specified:
* __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
* C++11 memory orders with the same names, see the C++11 standard or
* the GCC wiki on atomic synchronization for detailed definition.
*/
static __rte_always_inline void
rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
int memorder);
/**
* Wait for *addr to be updated with a 64-bit expected value, with a relaxed
* memory ordering model meaning the loads around this API can be reordered.
*
* @param addr
* A pointer to the memory location.
* @param expected
* A 64-bit expected value to be in the memory location.
* @param memorder
* Two different memory orders that can be specified:
* __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
* C++11 memory orders with the same names, see the C++11 standard or
* the GCC wiki on atomic synchronization for detailed definition.
*/
static __rte_always_inline void
rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
int memorder);
#ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
static __rte_always_inline void
rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
int memorder)
{
assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
while (__atomic_load_n(addr, memorder) != expected)
rte_pause();
}
static __rte_always_inline void
rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
int memorder)
{
assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
while (__atomic_load_n(addr, memorder) != expected)
rte_pause();
}
static __rte_always_inline void
rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
int memorder)
{
assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
while (__atomic_load_n(addr, memorder) != expected)
rte_pause();
}
/*
* Wait until *addr & mask makes the condition true. With a relaxed memory
* ordering model, the loads around this helper can be reordered.
*
* @param addr
* A pointer to the memory location.
* @param mask
* A mask of value bits in interest.
* @param cond
* A symbol representing the condition.
* @param expected
* An expected value to be in the memory location.
* @param memorder
* Two different memory orders that can be specified:
* __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
* C++11 memory orders with the same names, see the C++11 standard or
* the GCC wiki on atomic synchronization for detailed definition.
*/
#define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \
RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && \
memorder != __ATOMIC_RELAXED); \
typeof(*(addr)) expected_value = (expected); \
while (!((__atomic_load_n((addr), (memorder)) & (mask)) cond \
expected_value)) \
rte_pause(); \
} while (0)
#endif /* ! RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED */
#endif /* _RTE_PAUSE_H_ */

167
Programs/hugePage/common/generic/rte_power_intrinsics.h Normal file
View File

@@ -0,0 +1,167 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2020 Intel Corporation
*/
#ifndef _RTE_POWER_INTRINSIC_H_
#define _RTE_POWER_INTRINSIC_H_
#include <inttypes.h>
#include "rte_compat.h"
#include "rte_spinlock.h"
/**
* @file
* Advanced power management operations.
*
* This file define APIs for advanced power management,
* which are architecture-dependent.
*/
/** Size of the opaque data in monitor condition */
#define RTE_POWER_MONITOR_OPAQUE_SZ 4
/**
* Callback definition for monitoring conditions. Callbacks with this signature
* will be used by `rte_power_monitor()` to check if the entering of power
* optimized state should be aborted.
*
* @param val
* The value read from memory.
* @param opaque
* Callback-specific data.
*
* @return
* 0 if entering of power optimized state should proceed
* -1 if entering of power optimized state should be aborted
*/
typedef int (*rte_power_monitor_clb_t)(const uint64_t val,
const uint64_t opaque[RTE_POWER_MONITOR_OPAQUE_SZ]);
struct rte_power_monitor_cond {
volatile void *addr; /**< Address to monitor for changes */
uint8_t size; /**< Data size (in bytes) that will be read from the
* monitored memory location (`addr`). Can be 1, 2,
* 4, or 8. Supplying any other value will result in
* an error.
*/
rte_power_monitor_clb_t fn; /**< Callback to be used to check if
* entering power optimized state should
* be aborted.
*/
uint64_t opaque[RTE_POWER_MONITOR_OPAQUE_SZ];
/**< Callback-specific data */
};
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice.
*
* Monitor specific address for changes. This will cause the CPU to enter an
* architecture-defined optimized power state until either the specified
* memory address is written to, a certain TSC timestamp is reached, or other
* reasons cause the CPU to wake up.
*
* Additionally, an expected value (`pmc->val`), mask (`pmc->mask`), and data
* size (`pmc->size`) are provided in the `pmc` power monitoring condition. If
* the mask is non-zero, the current value pointed to by the `pmc->addr` pointer
* will be read and compared against the expected value, and if they match, the
* entering of optimized power state will be aborted. This is intended to
* prevent the CPU from entering optimized power state and waiting on a write
* that has already happened by the time this API is called.
*
* @warning It is responsibility of the user to check if this function is
* supported at runtime using `rte_cpu_get_intrinsics_support()` API call.
*
* @param pmc
* The monitoring condition structure.
* @param tsc_timestamp
* Maximum TSC timestamp to wait for. Note that the wait behavior is
* architecture-dependent.
*
* @return
* 0 on success
* -EINVAL on invalid parameters
* -ENOTSUP if unsupported
*/
__rte_experimental
int rte_power_monitor(const struct rte_power_monitor_cond *pmc,
const uint64_t tsc_timestamp);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice.
*
* Wake up a specific lcore that is in a power optimized state and is monitoring
* an address.
*
* @note It is safe to call this function if the lcore in question is not
* sleeping. The function will have no effect.
*
* @note This function will *not* wake up a core that is in a power optimized
* state due to calling `rte_power_pause`.
*
* @param lcore_id
* Lcore ID of a sleeping thread.
*/
__rte_experimental
int rte_power_monitor_wakeup(const unsigned int lcore_id);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice.
*
* Enter an architecture-defined optimized power state until a certain TSC
* timestamp is reached.
*
* @warning It is responsibility of the user to check if this function is
* supported at runtime using `rte_cpu_get_intrinsics_support()` API call.
*
* @param tsc_timestamp
* Maximum TSC timestamp to wait for. Note that the wait behavior is
* architecture-dependent.
*
* @return
* 0 on success
* -EINVAL on invalid parameters
* -ENOTSUP if unsupported
*/
__rte_experimental
int rte_power_pause(const uint64_t tsc_timestamp);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Monitor a set of addresses for changes. This will cause the CPU to enter an
* architecture-defined optimized power state until either one of the specified
* memory addresses is written to, a certain TSC timestamp is reached, or other
* reasons cause the CPU to wake up.
*
* Additionally, `expected` 64-bit values and 64-bit masks are provided. If
* mask is non-zero, the current value pointed to by the `p` pointer will be
* checked against the expected value, and if they do not match, the entering of
* optimized power state may be aborted.
*
* @warning It is responsibility of the user to check if this function is
* supported at runtime using `rte_cpu_get_intrinsics_support()` API call.
* Failing to do so may result in an illegal CPU instruction error.
*
* @param pmc
* An array of monitoring condition structures.
* @param num
* Length of the `pmc` array.
* @param tsc_timestamp
* Maximum TSC timestamp to wait for. Note that the wait behavior is
* architecture-dependent.
*
* @return
* 0 on success
* -EINVAL on invalid parameters
* -ENOTSUP if unsupported
*/
__rte_experimental
int rte_power_monitor_multi(const struct rte_power_monitor_cond pmc[],
const uint32_t num, const uint64_t tsc_timestamp);
#endif /* _RTE_POWER_INTRINSIC_H_ */

137
Programs/hugePage/common/generic/rte_prefetch.h Normal file
View File

@@ -0,0 +1,137 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2015 Intel Corporation
*/
#ifndef _RTE_PREFETCH_H_
#define _RTE_PREFETCH_H_
#include "rte_compat.h"
/**
* @file
*
* Prefetch operations.
*
* This file defines an API for prefetch macros / inline-functions,
* which are architecture-dependent. Prefetching occurs when a
* processor requests an instruction or data from memory to cache
* before it is actually needed, potentially speeding up the execution of the
* program.
*/
/**
* Prefetch a cache line into all cache levels.
* @param p
* Address to prefetch
*/
static inline void rte_prefetch0(const volatile void *p);
/**
* Prefetch a cache line into all cache levels except the 0th cache level.
* @param p
* Address to prefetch
*/
static inline void rte_prefetch1(const volatile void *p);
/**
* Prefetch a cache line into all cache levels except the 0th and 1th cache
* levels.
* @param p
* Address to prefetch
*/
static inline void rte_prefetch2(const volatile void *p);
/**
* Prefetch a cache line into all cache levels (non-temporal/transient version)
*
* The non-temporal prefetch is intended as a prefetch hint that processor will
* use the prefetched data only once or short period, unlike the
* rte_prefetch0() function which imply that prefetched data to use repeatedly.
*
* @param p
* Address to prefetch
*/
static inline void rte_prefetch_non_temporal(const volatile void *p);
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Prefetch a cache line into all cache levels, with intention to write. This
* prefetch variant hints to the CPU that the program is expecting to write to
* the cache line being prefetched.
*
* @param p Address to prefetch
*/
__rte_experimental
static inline void
rte_prefetch0_write(const void *p)
{
/* 1 indicates intention to write, 3 sets target cache level to L1. See
* GCC docs where these integer constants are described in more detail:
* https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
*/
__builtin_prefetch(p, 1, 3);
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Prefetch a cache line into all cache levels, except the 0th, with intention
* to write. This prefetch variant hints to the CPU that the program is
* expecting to write to the cache line being prefetched.
*
* @param p Address to prefetch
*/
__rte_experimental
static inline void
rte_prefetch1_write(const void *p)
{
/* 1 indicates intention to write, 2 sets target cache level to L2. See
* GCC docs where these integer constants are described in more detail:
* https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
*/
__builtin_prefetch(p, 1, 2);
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Prefetch a cache line into all cache levels, except the 0th and 1st, with
* intention to write. This prefetch variant hints to the CPU that the program
* is expecting to write to the cache line being prefetched.
*
* @param p Address to prefetch
*/
__rte_experimental
static inline void
rte_prefetch2_write(const void *p)
{
/* 1 indicates intention to write, 1 sets target cache level to L3. See
* GCC docs where these integer constants are described in more detail:
* https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
*/
__builtin_prefetch(p, 1, 1);
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Demote a cache line to a more distant level of cache from the processor.
* CLDEMOTE hints to hardware to move (demote) a cache line from the closest to
* the processor to a level more distant from the processor. It is a hint and
* not guaranteed. rte_cldemote is intended to move the cache line to the more
* remote cache, where it expects sharing to be efficient and to indicate that
* a line may be accessed by a different core in the future.
*
* @param p
* Address to demote
*/
__rte_experimental
static inline void
rte_cldemote(const volatile void *p);
#endif /* _RTE_PREFETCH_H_ */

280
Programs/hugePage/common/generic/rte_rwlock.h Normal file
View File

@@ -0,0 +1,280 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_RWLOCK_H_
#define _RTE_RWLOCK_H_
/**
* @file
*
* RTE Read-Write Locks
*
* This file defines an API for read-write locks. The lock is used to
* protect data that allows multiple readers in parallel, but only
* one writer. All readers are blocked until the writer is finished
* writing.
*
* This version does not give preference to readers or writers
* and does not starve either readers or writers.
*
* See also:
* https://locklessinc.com/articles/locks/
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <errno.h>
#include "../rte_branch_prediction.h"
#include "../rte_common.h"
#include "rte_pause.h"
/**
* The rte_rwlock_t type.
*
* Readers increment the counter by RTE_RWLOCK_READ (4)
* Writers set the RTE_RWLOCK_WRITE bit when lock is held
* and set the RTE_RWLOCK_WAIT bit while waiting.
*
* 31 2 1 0
* +-------------------+-+-+
* | readers | | |
* +-------------------+-+-+
* ^ ^
* | |
* WRITE: lock held ----/ |
* WAIT: writer pending --/
*/
#define RTE_RWLOCK_WAIT 0x1 /* Writer is waiting */
#define RTE_RWLOCK_WRITE 0x2 /* Writer has the lock */
#define RTE_RWLOCK_MASK (RTE_RWLOCK_WAIT | RTE_RWLOCK_WRITE)
/* Writer is waiting or has lock */
#define RTE_RWLOCK_READ 0x4 /* Reader increment */
typedef struct {
int32_t cnt;
} rte_rwlock_t;
/**
* A static rwlock initializer.
*/
#define RTE_RWLOCK_INITIALIZER { 0 }
/**
* Initialize the rwlock to an unlocked state.
*
* @param rwl
* A pointer to the rwlock structure.
*/
static inline void
rte_rwlock_init(rte_rwlock_t *rwl)
{
rwl->cnt = 0;
}
/**
* Take a read lock. Loop until the lock is held.
*
* @param rwl
* A pointer to a rwlock structure.
*/
static inline void
rte_rwlock_read_lock(rte_rwlock_t *rwl)
{
int32_t x;
while (1) {
/* Wait while writer is present or pending */
while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED)
& RTE_RWLOCK_MASK)
rte_pause();
/* Try to get read lock */
x = __atomic_add_fetch(&rwl->cnt, RTE_RWLOCK_READ,
__ATOMIC_ACQUIRE);
/* If no writer, then acquire was successful */
if (likely(!(x & RTE_RWLOCK_MASK)))
return;
/* Lost race with writer, backout the change. */
__atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ,
__ATOMIC_RELAXED);
}
}
/**
* Try to take a read lock.
*
* @param rwl
* A pointer to a rwlock structure.
* @return
* - zero if the lock is successfully taken
* - -EBUSY if lock could not be acquired for reading because a
* writer holds the lock
*/
static inline int
rte_rwlock_read_trylock(rte_rwlock_t *rwl)
{
int32_t x;
x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
/* fail if write lock is held or writer is pending */
if (x & RTE_RWLOCK_MASK)
return -EBUSY;
/* Try to get read lock */
x = __atomic_add_fetch(&rwl->cnt, RTE_RWLOCK_READ,
__ATOMIC_ACQUIRE);
/* Back out if writer raced in */
if (unlikely(x & RTE_RWLOCK_MASK)) {
__atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ,
__ATOMIC_RELEASE);
return -EBUSY;
}
return 0;
}
/**
* Release a read lock.
*
* @param rwl
* A pointer to the rwlock structure.
*/
static inline void
rte_rwlock_read_unlock(rte_rwlock_t *rwl)
{
__atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ, __ATOMIC_RELEASE);
}
/**
* Try to take a write lock.
*
* @param rwl
* A pointer to a rwlock structure.
* @return
* - zero if the lock is successfully taken
* - -EBUSY if lock could not be acquired for writing because
* it was already locked for reading or writing
*/
static inline int
rte_rwlock_write_trylock(rte_rwlock_t *rwl)
{
int32_t x;
x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
if (x < RTE_RWLOCK_WRITE &&
__atomic_compare_exchange_n(&rwl->cnt, &x, x + RTE_RWLOCK_WRITE,
1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
return 0;
else
return -EBUSY;
}
/**
* Take a write lock. Loop until the lock is held.
*
* @param rwl
* A pointer to a rwlock structure.
*/
static inline void
rte_rwlock_write_lock(rte_rwlock_t *rwl)
{
int32_t x;
while (1) {
x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
/* No readers or writers? */
if (likely(x < RTE_RWLOCK_WRITE)) {
/* Turn off RTE_RWLOCK_WAIT, turn on RTE_RWLOCK_WRITE */
if (__atomic_compare_exchange_n(&rwl->cnt, &x, RTE_RWLOCK_WRITE, 1,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
return;
}
/* Turn on writer wait bit */
if (!(x & RTE_RWLOCK_WAIT))
__atomic_fetch_or(&rwl->cnt, RTE_RWLOCK_WAIT, __ATOMIC_RELAXED);
/* Wait until no readers before trying again */
while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) > RTE_RWLOCK_WAIT)
rte_pause();
}
}
/**
* Release a write lock.
*
* @param rwl
* A pointer to a rwlock structure.
*/
static inline void
rte_rwlock_write_unlock(rte_rwlock_t *rwl)
{
__atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_WRITE, __ATOMIC_RELEASE);
}
/**
* Try to execute critical section in a hardware memory transaction, if it
* fails or not available take a read lock
*
* NOTE: An attempt to perform a HW I/O operation inside a hardware memory
* transaction always aborts the transaction since the CPU is not able to
* roll-back should the transaction fail. Therefore, hardware transactional
* locks are not advised to be used around rte_eth_rx_burst() and
* rte_eth_tx_burst() calls.
*
* @param rwl
* A pointer to a rwlock structure.
*/
static inline void
rte_rwlock_read_lock_tm(rte_rwlock_t *rwl);
/**
* Commit hardware memory transaction or release the read lock if the lock is used as a fall-back
*
* @param rwl
* A pointer to the rwlock structure.
*/
static inline void
rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl);
/**
* Try to execute critical section in a hardware memory transaction, if it
* fails or not available take a write lock
*
* NOTE: An attempt to perform a HW I/O operation inside a hardware memory
* transaction always aborts the transaction since the CPU is not able to
* roll-back should the transaction fail. Therefore, hardware transactional
* locks are not advised to be used around rte_eth_rx_burst() and
* rte_eth_tx_burst() calls.
*
* @param rwl
* A pointer to a rwlock structure.
*/
static inline void
rte_rwlock_write_lock_tm(rte_rwlock_t *rwl);
/**
* Commit hardware memory transaction or release the write lock if the lock is used as a fall-back
*
* @param rwl
* A pointer to a rwlock structure.
*/
static inline void
rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl);
#ifdef __cplusplus
}
#endif
#endif /* _RTE_RWLOCK_H_ */

309
Programs/hugePage/common/generic/rte_spinlock.h Normal file
View File

@@ -0,0 +1,309 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_SPINLOCK_H_
#define _RTE_SPINLOCK_H_
/**
* @file
*
* RTE Spinlocks
*
* This file defines an API for read-write locks, which are implemented
* in an architecture-specific way. This kind of lock simply waits in
* a loop repeatedly checking until the lock becomes available.
*
* All locks must be initialised before use, and only initialised once.
*
*/
#include "rte_lcore.h"
#ifdef RTE_FORCE_INTRINSICS
#include "rte_common.h"
#endif
#include "rte_pause.h"
/**
* The rte_spinlock_t type.
*/
typedef struct {
volatile int locked; /**< lock status 0 = unlocked, 1 = locked */
} rte_spinlock_t;
/**
* A static spinlock initializer.
*/
#define RTE_SPINLOCK_INITIALIZER { 0 }
/**
* Initialize the spinlock to an unlocked state.
*
* @param sl
* A pointer to the spinlock.
*/
static inline void
rte_spinlock_init(rte_spinlock_t *sl)
{
sl->locked = 0;
}
/**
* Take the spinlock.
*
* @param sl
* A pointer to the spinlock.
*/
static inline void
rte_spinlock_lock(rte_spinlock_t *sl);
#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_spinlock_lock(rte_spinlock_t *sl)
{
int exp = 0;
while (!__atomic_compare_exchange_n(&sl->locked, &exp, 1, 0,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
rte_wait_until_equal_32((volatile uint32_t *)&sl->locked,
0, __ATOMIC_RELAXED);
exp = 0;
}
}
#endif
/**
* Release the spinlock.
*
* @param sl
* A pointer to the spinlock.
*/
static inline void
rte_spinlock_unlock (rte_spinlock_t *sl);
#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_spinlock_unlock (rte_spinlock_t *sl)
{
__atomic_store_n(&sl->locked, 0, __ATOMIC_RELEASE);
}
#endif
/**
* Try to take the lock.
*
* @param sl
* A pointer to the spinlock.
* @return
* 1 if the lock is successfully taken; 0 otherwise.
*/
__rte_warn_unused_result
static inline int
rte_spinlock_trylock (rte_spinlock_t *sl);
#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_spinlock_trylock (rte_spinlock_t *sl)
{
int exp = 0;
return __atomic_compare_exchange_n(&sl->locked, &exp, 1,
0, /* disallow spurious failure */
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
#endif
/**
* Test if the lock is taken.
*
* @param sl
* A pointer to the spinlock.
* @return
* 1 if the lock is currently taken; 0 otherwise.
*/
static inline int rte_spinlock_is_locked (rte_spinlock_t *sl)
{
return __atomic_load_n(&sl->locked, __ATOMIC_ACQUIRE);
}
/**
* Test if hardware transactional memory (lock elision) is supported
*
* @return
* 1 if the hardware transactional memory is supported; 0 otherwise.
*/
static inline int rte_tm_supported(void);
/**
* Try to execute critical section in a hardware memory transaction,
* if it fails or not available take the spinlock.
*
* NOTE: An attempt to perform a HW I/O operation inside a hardware memory
* transaction always aborts the transaction since the CPU is not able to
* roll-back should the transaction fail. Therefore, hardware transactional
* locks are not advised to be used around rte_eth_rx_burst() and
* rte_eth_tx_burst() calls.
*
* @param sl
* A pointer to the spinlock.
*/
static inline void
rte_spinlock_lock_tm(rte_spinlock_t *sl);
/**
* Commit hardware memory transaction or release the spinlock if
* the spinlock is used as a fall-back
*
* @param sl
* A pointer to the spinlock.
*/
static inline void
rte_spinlock_unlock_tm(rte_spinlock_t *sl);
/**
* Try to execute critical section in a hardware memory transaction,
* if it fails or not available try to take the lock.
*
* NOTE: An attempt to perform a HW I/O operation inside a hardware memory
* transaction always aborts the transaction since the CPU is not able to
* roll-back should the transaction fail. Therefore, hardware transactional
* locks are not advised to be used around rte_eth_rx_burst() and
* rte_eth_tx_burst() calls.
*
* @param sl
* A pointer to the spinlock.
* @return
* 1 if the hardware memory transaction is successfully started
* or lock is successfully taken; 0 otherwise.
*/
__rte_warn_unused_result
static inline int
rte_spinlock_trylock_tm(rte_spinlock_t *sl);
/**
* The rte_spinlock_recursive_t type.
*/
typedef struct {
rte_spinlock_t sl; /**< the actual spinlock */
volatile int user; /**< core id using lock, -1 for unused */
volatile int count; /**< count of time this lock has been called */
} rte_spinlock_recursive_t;
/**
* A static recursive spinlock initializer.
*/
#define RTE_SPINLOCK_RECURSIVE_INITIALIZER {RTE_SPINLOCK_INITIALIZER, -1, 0}
/**
* Initialize the recursive spinlock to an unlocked state.
*
* @param slr
* A pointer to the recursive spinlock.
*/
static inline void rte_spinlock_recursive_init(rte_spinlock_recursive_t *slr)
{
rte_spinlock_init(&slr->sl);
slr->user = -1;
slr->count = 0;
}
/**
* Take the recursive spinlock.
*
* @param slr
* A pointer to the recursive spinlock.
*/
static inline void rte_spinlock_recursive_lock(rte_spinlock_recursive_t *slr)
{
int id = rte_gettid();
if (slr->user != id) {
rte_spinlock_lock(&slr->sl);
slr->user = id;
}
slr->count++;
}
/**
* Release the recursive spinlock.
*
* @param slr
* A pointer to the recursive spinlock.
*/
static inline void rte_spinlock_recursive_unlock(rte_spinlock_recursive_t *slr)
{
if (--(slr->count) == 0) {
slr->user = -1;
rte_spinlock_unlock(&slr->sl);
}
}
/**
* Try to take the recursive lock.
*
* @param slr
* A pointer to the recursive spinlock.
* @return
* 1 if the lock is successfully taken; 0 otherwise.
*/
__rte_warn_unused_result
static inline int rte_spinlock_recursive_trylock(rte_spinlock_recursive_t *slr)
{
int id = rte_gettid();
if (slr->user != id) {
if (rte_spinlock_trylock(&slr->sl) == 0)
return 0;
slr->user = id;
}
slr->count++;
return 1;
}
/**
* Try to execute critical section in a hardware memory transaction,
* if it fails or not available take the recursive spinlocks
*
* NOTE: An attempt to perform a HW I/O operation inside a hardware memory
* transaction always aborts the transaction since the CPU is not able to
* roll-back should the transaction fail. Therefore, hardware transactional
* locks are not advised to be used around rte_eth_rx_burst() and
* rte_eth_tx_burst() calls.
*
* @param slr
* A pointer to the recursive spinlock.
*/
static inline void rte_spinlock_recursive_lock_tm(
rte_spinlock_recursive_t *slr);
/**
* Commit hardware memory transaction or release the recursive spinlock
* if the recursive spinlock is used as a fall-back
*
* @param slr
* A pointer to the recursive spinlock.
*/
static inline void rte_spinlock_recursive_unlock_tm(
rte_spinlock_recursive_t *slr);
/**
* Try to execute critical section in a hardware memory transaction,
* if it fails or not available try to take the recursive lock
*
* NOTE: An attempt to perform a HW I/O operation inside a hardware memory
* transaction always aborts the transaction since the CPU is not able to
* roll-back should the transaction fail. Therefore, hardware transactional
* locks are not advised to be used around rte_eth_rx_burst() and
* rte_eth_tx_burst() calls.
*
* @param slr
* A pointer to the recursive spinlock.
* @return
* 1 if the hardware memory transaction is successfully started
* or lock is successfully taken; 0 otherwise.
*/
__rte_warn_unused_result
static inline int rte_spinlock_recursive_trylock_tm(
rte_spinlock_recursive_t *slr);
#endif /* _RTE_SPINLOCK_H_ */

235
Programs/hugePage/common/generic/rte_vect.h Normal file
View File

@@ -0,0 +1,235 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2016 6WIND S.A.
*/
#ifndef _RTE_VECT_H_
#define _RTE_VECT_H_
/**
* @file
* SIMD vector types and control
*
* This file defines types to use vector instructions with generic C code
* and APIs to enable the code using them.
*/
#include <stdint.h>
#include "rte_compat.h"
/* Unsigned vector types */
/**
* 64 bits vector size to use with unsigned 8 bits elements.
*
* a = (rte_v64u8_t){ a0, a1, a2, a3, a4, a5, a6, a7 }
*/
typedef uint8_t rte_v64u8_t __attribute__((vector_size(8), aligned(8)));
/**
* 64 bits vector size to use with unsigned 16 bits elements.
*
* a = (rte_v64u16_t){ a0, a1, a2, a3 }
*/
typedef uint16_t rte_v64u16_t __attribute__((vector_size(8), aligned(8)));
/**
* 64 bits vector size to use with unsigned 32 bits elements.
*
* a = (rte_v64u32_t){ a0, a1 }
*/
typedef uint32_t rte_v64u32_t __attribute__((vector_size(8), aligned(8)));
/**
* 128 bits vector size to use with unsigned 8 bits elements.
*
* a = (rte_v128u8_t){ a00, a01, a02, a03, a04, a05, a06, a07,
* a08, a09, a10, a11, a12, a13, a14, a15 }
*/
typedef uint8_t rte_v128u8_t __attribute__((vector_size(16), aligned(16)));
/**
* 128 bits vector size to use with unsigned 16 bits elements.
*
* a = (rte_v128u16_t){ a0, a1, a2, a3, a4, a5, a6, a7 }
*/
typedef uint16_t rte_v128u16_t __attribute__((vector_size(16), aligned(16)));
/**
* 128 bits vector size to use with unsigned 32 bits elements.
*
* a = (rte_v128u32_t){ a0, a1, a2, a3 }
*/
typedef uint32_t rte_v128u32_t __attribute__((vector_size(16), aligned(16)));
/**
* 128 bits vector size to use with unsigned 64 bits elements.
*
* a = (rte_v128u64_t){ a0, a1 }
*/
typedef uint64_t rte_v128u64_t __attribute__((vector_size(16), aligned(16)));
/**
* 256 bits vector size to use with unsigned 8 bits elements.
*
* a = (rte_v256u8_t){ a00, a01, a02, a03, a04, a05, a06, a07,
* a08, a09, a10, a11, a12, a13, a14, a15,
* a16, a17, a18, a19, a20, a21, a22, a23,
* a24, a25, a26, a27, a28, a29, a30, a31 }
*/
typedef uint8_t rte_v256u8_t __attribute__((vector_size(32), aligned(32)));
/**
* 256 bits vector size to use with unsigned 16 bits elements.
*
* a = (rte_v256u16_t){ a00, a01, a02, a03, a04, a05, a06, a07,
* a08, a09, a10, a11, a12, a13, a14, a15 }
*/
typedef uint16_t rte_v256u16_t __attribute__((vector_size(32), aligned(32)));
/**
* 256 bits vector size to use with unsigned 32 bits elements.
*
* a = (rte_v256u32_t){ a0, a1, a2, a3, a4, a5, a6, a7 }
*/
typedef uint32_t rte_v256u32_t __attribute__((vector_size(32), aligned(32)));
/**
* 256 bits vector size to use with unsigned 64 bits elements.
*
* a = (rte_v256u64_t){ a0, a1, a2, a3 }
*/
typedef uint64_t rte_v256u64_t __attribute__((vector_size(32), aligned(32)));
/* Signed vector types */
/**
* 64 bits vector size to use with 8 bits elements.
*
* a = (rte_v64s8_t){ a0, a1, a2, a3, a4, a5, a6, a7 }
*/
typedef int8_t rte_v64s8_t __attribute__((vector_size(8), aligned(8)));
/**
* 64 bits vector size to use with 16 bits elements.
*
* a = (rte_v64s16_t){ a0, a1, a2, a3 }
*/
typedef int16_t rte_v64s16_t __attribute__((vector_size(8), aligned(8)));
/**
* 64 bits vector size to use with 32 bits elements.
*
* a = (rte_v64s32_t){ a0, a1 }
*/
typedef int32_t rte_v64s32_t __attribute__((vector_size(8), aligned(8)));
/**
* 128 bits vector size to use with 8 bits elements.
*
* a = (rte_v128s8_t){ a00, a01, a02, a03, a04, a05, a06, a07,
* a08, a09, a10, a11, a12, a13, a14, a15 }
*/
typedef int8_t rte_v128s8_t __attribute__((vector_size(16), aligned(16)));
/**
* 128 bits vector size to use with 16 bits elements.
*
* a = (rte_v128s16_t){ a0, a1, a2, a3, a4, a5, a6, a7 }
*/
typedef int16_t rte_v128s16_t __attribute__((vector_size(16), aligned(16)));
/**
* 128 bits vector size to use with 32 bits elements.
*
* a = (rte_v128s32_t){ a0, a1, a2, a3 }
*/
typedef int32_t rte_v128s32_t __attribute__((vector_size(16), aligned(16)));
/**
* 128 bits vector size to use with 64 bits elements.
*
* a = (rte_v128s64_t){ a1, a2 }
*/
typedef int64_t rte_v128s64_t __attribute__((vector_size(16), aligned(16)));
/**
* 256 bits vector size to use with 8 bits elements.
*
* a = (rte_v256s8_t){ a00, a01, a02, a03, a04, a05, a06, a07,
* a08, a09, a10, a11, a12, a13, a14, a15,
* a16, a17, a18, a19, a20, a21, a22, a23,
* a24, a25, a26, a27, a28, a29, a30, a31 }
*/
typedef int8_t rte_v256s8_t __attribute__((vector_size(32), aligned(32)));
/**
* 256 bits vector size to use with 16 bits elements.
*
* a = (rte_v256s16_t){ a00, a01, a02, a03, a04, a05, a06, a07,
* a08, a09, a10, a11, a12, a13, a14, a15 }
*/
typedef int16_t rte_v256s16_t __attribute__((vector_size(32), aligned(32)));
/**
* 256 bits vector size to use with 32 bits elements.
*
* a = (rte_v256s32_t){ a0, a1, a2, a3, a4, a5, a6, a7 }
*/
typedef int32_t rte_v256s32_t __attribute__((vector_size(32), aligned(32)));
/**
* 256 bits vector size to use with 64 bits elements.
*
* a = (rte_v256s64_t){ a0, a1, a2, a3 }
*/
typedef int64_t rte_v256s64_t __attribute__((vector_size(32), aligned(32)));
/**
* The max SIMD bitwidth value to limit vector path selection.
*/
enum rte_vect_max_simd {
RTE_VECT_SIMD_DISABLED = 64,
/**< Limits path selection to scalar, disables all vector paths. */
RTE_VECT_SIMD_128 = 128,
/**< Limits path selection to SSE/NEON/Altivec or below. */
RTE_VECT_SIMD_256 = 256, /**< Limits path selection to AVX2 or below. */
RTE_VECT_SIMD_512 = 512, /**< Limits path selection to AVX512 or below. */
RTE_VECT_SIMD_MAX = INT16_MAX + 1,
/**<
* Disables limiting by max SIMD bitwidth, allows all suitable paths.
* This value is used as it is a large number and a power of 2.
*/
};
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Get the supported SIMD bitwidth.
*
* @return
* uint16_t bitwidth.
*/
__rte_experimental
uint16_t rte_vect_get_max_simd_bitwidth(void);
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Set the supported SIMD bitwidth.
* This API should only be called once at initialization, before EAL init.
*
* @param bitwidth
* uint16_t bitwidth.
* @return
* - 0 on success.
* - -EINVAL on invalid bitwidth parameter.
* - -EPERM if bitwidth is forced.
*/
__rte_experimental
int rte_vect_set_max_simd_bitwidth(uint16_t bitwidth);
#endif /* _RTE_VECT_H_ */

471
Programs/hugePage/common/hotplug_mp.c Normal file
View File

@@ -0,0 +1,471 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Intel Corporation
*/
#include <stdlib.h>
#include <string.h>
#include <bus_driver.h>
#include "rte_eal.h"
#include "rte_errno.h"
#include "rte_alarm.h"
#include "rte_string_fns.h"
#include "rte_devargs.h"
#include "hotplug_mp.h"
#include "eal_private.h"
#define MP_TIMEOUT_S 5 /**< 5 seconds timeouts */
struct mp_reply_bundle {
struct rte_mp_msg msg;
void *peer;
};
static int cmp_dev_name(const struct rte_device *dev, const void *_name)
{
const char *name = _name;
return strcmp(dev->name, name);
}
/**
* Secondary to primary request.
* start from function eal_dev_hotplug_request_to_primary.
*
* device attach on secondary:
* a) secondary send sync request to the primary.
* b) primary receive the request and attach the new device if
* failed goto i).
* c) primary forward attach sync request to all secondary.
* d) secondary receive the request and attach the device and send a reply.
* e) primary check the reply if all success goes to j).
* f) primary send attach rollback sync request to all secondary.
* g) secondary receive the request and detach the device and send a reply.
* h) primary receive the reply and detach device as rollback action.
* i) send attach fail to secondary as a reply of step a), goto k).
* j) send attach success to secondary as a reply of step a).
* k) secondary receive reply and return.
*
* device detach on secondary:
* a) secondary send sync request to the primary.
* b) primary send detach sync request to all secondary.
* c) secondary detach the device and send a reply.
* d) primary check the reply if all success goes to g).
* e) primary send detach rollback sync request to all secondary.
* f) secondary receive the request and attach back device. goto h).
* g) primary detach the device if success goto i), else goto e).
* h) primary send detach fail to secondary as a reply of step a), goto j).
* i) primary send detach success to secondary as a reply of step a).
* j) secondary receive reply and return.
*/
static int
send_response_to_secondary(const struct eal_dev_mp_req *req,
int result,
const void *peer)
{
struct rte_mp_msg mp_resp;
struct eal_dev_mp_req *resp =
(struct eal_dev_mp_req *)mp_resp.param;
int ret;
memset(&mp_resp, 0, sizeof(mp_resp));
mp_resp.len_param = sizeof(*resp);
strlcpy(mp_resp.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_resp.name));
memcpy(resp, req, sizeof(*req));
resp->result = result;
ret = rte_mp_reply(&mp_resp, peer);
if (ret != 0)
RTE_LOG(ERR, EAL, "failed to send response to secondary\n");
return ret;
}
static void
__handle_secondary_request(void *param)
{
struct mp_reply_bundle *bundle = param;
const struct rte_mp_msg *msg = &bundle->msg;
const struct eal_dev_mp_req *req =
(const struct eal_dev_mp_req *)msg->param;
struct eal_dev_mp_req tmp_req;
struct rte_devargs da;
struct rte_device *dev;
struct rte_bus *bus;
int ret = 0;
tmp_req = *req;
memset(&da, 0, sizeof(da));
if (req->t == EAL_DEV_REQ_TYPE_ATTACH) {
ret = local_dev_probe(req->devargs, &dev);
if (ret != 0 && ret != -EEXIST) {
RTE_LOG(ERR, EAL, "Failed to hotplug add device on primary\n");
goto finish;
}
ret = eal_dev_hotplug_request_to_secondary(&tmp_req);
if (ret != 0) {
RTE_LOG(ERR, EAL, "Failed to send hotplug request to secondary\n");
ret = -ENOMSG;
goto rollback;
}
if (tmp_req.result != 0) {
ret = tmp_req.result;
RTE_LOG(ERR, EAL, "Failed to hotplug add device on secondary\n");
if (ret != -EEXIST)
goto rollback;
}
} else if (req->t == EAL_DEV_REQ_TYPE_DETACH) {
ret = rte_devargs_parse(&da, req->devargs);
if (ret != 0)
goto finish;
ret = eal_dev_hotplug_request_to_secondary(&tmp_req);
if (ret != 0) {
RTE_LOG(ERR, EAL, "Failed to send hotplug request to secondary\n");
ret = -ENOMSG;
goto rollback;
}
bus = rte_bus_find_by_name(da.bus->name);
if (bus == NULL) {
RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", da.bus->name);
ret = -ENOENT;
goto finish;
}
dev = bus->find_device(NULL, cmp_dev_name, da.name);
if (dev == NULL) {
RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", da.name);
ret = -ENOENT;
goto finish;
}
if (tmp_req.result != 0) {
RTE_LOG(ERR, EAL, "Failed to hotplug remove device on secondary\n");
ret = tmp_req.result;
if (ret != -ENOENT)
goto rollback;
}
ret = local_dev_remove(dev);
if (ret != 0) {
RTE_LOG(ERR, EAL, "Failed to hotplug remove device on primary\n");
if (ret != -ENOENT)
goto rollback;
}
} else {
RTE_LOG(ERR, EAL, "unsupported secondary to primary request\n");
ret = -ENOTSUP;
}
goto finish;
rollback:
if (req->t == EAL_DEV_REQ_TYPE_ATTACH) {
tmp_req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK;
eal_dev_hotplug_request_to_secondary(&tmp_req);
local_dev_remove(dev);
} else {
tmp_req.t = EAL_DEV_REQ_TYPE_DETACH_ROLLBACK;
eal_dev_hotplug_request_to_secondary(&tmp_req);
}
finish:
ret = send_response_to_secondary(&tmp_req, ret, bundle->peer);
if (ret)
RTE_LOG(ERR, EAL, "failed to send response to secondary\n");
rte_devargs_reset(&da);
free(bundle->peer);
free(bundle);
}
static int
handle_secondary_request(const struct rte_mp_msg *msg, const void *peer)
{
struct mp_reply_bundle *bundle;
const struct eal_dev_mp_req *req =
(const struct eal_dev_mp_req *)msg->param;
int ret = 0;
bundle = malloc(sizeof(*bundle));
if (bundle == NULL) {
RTE_LOG(ERR, EAL, "not enough memory\n");
return send_response_to_secondary(req, -ENOMEM, peer);
}
bundle->msg = *msg;
/**
* We need to send reply on interrupt thread, but peer can't be
* parsed directly, so this is a temporal hack, need to be fixed
* when it is ready.
*/
bundle->peer = strdup(peer);
if (bundle->peer == NULL) {
free(bundle);
RTE_LOG(ERR, EAL, "not enough memory\n");
return send_response_to_secondary(req, -ENOMEM, peer);
}
/**
* We are at IPC callback thread, sync IPC is not allowed due to
* dead lock, so we delegate the task to interrupt thread.
*/
ret = rte_eal_alarm_set(1, __handle_secondary_request, bundle);
if (ret != 0) {
RTE_LOG(ERR, EAL, "failed to add mp task\n");
free(bundle->peer);
free(bundle);
return send_response_to_secondary(req, ret, peer);
}
return 0;
}
static void __handle_primary_request(void *param)
{
struct mp_reply_bundle *bundle = param;
struct rte_mp_msg *msg = &bundle->msg;
const struct eal_dev_mp_req *req =
(const struct eal_dev_mp_req *)msg->param;
struct rte_mp_msg mp_resp;
struct eal_dev_mp_req *resp =
(struct eal_dev_mp_req *)mp_resp.param;
struct rte_devargs *da;
struct rte_device *dev;
struct rte_bus *bus;
int ret = 0;
memset(&mp_resp, 0, sizeof(mp_resp));
switch (req->t) {
case EAL_DEV_REQ_TYPE_ATTACH:
case EAL_DEV_REQ_TYPE_DETACH_ROLLBACK:
ret = local_dev_probe(req->devargs, &dev);
break;
case EAL_DEV_REQ_TYPE_DETACH:
case EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK:
da = calloc(1, sizeof(*da));
if (da == NULL) {
ret = -ENOMEM;
break;
}
ret = rte_devargs_parse(da, req->devargs);
if (ret != 0)
goto quit;
bus = rte_bus_find_by_name(da->bus->name);
if (bus == NULL) {
RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", da->bus->name);
ret = -ENOENT;
goto quit;
}
dev = bus->find_device(NULL, cmp_dev_name, da->name);
if (dev == NULL) {
RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", da->name);
ret = -ENOENT;
goto quit;
}
if (!rte_dev_is_probed(dev)) {
if (req->t == EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK) {
/**
* Don't fail the rollback just because there's
* nothing to do.
*/
ret = 0;
} else
ret = -ENODEV;
goto quit;
}
ret = local_dev_remove(dev);
quit:
rte_devargs_reset(da);
free(da);
break;
default:
ret = -EINVAL;
}
strlcpy(mp_resp.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_resp.name));
mp_resp.len_param = sizeof(*req);
memcpy(resp, req, sizeof(*resp));
resp->result = ret;
if (rte_mp_reply(&mp_resp, bundle->peer) < 0)
RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
free(bundle->peer);
free(bundle);
}
static int
handle_primary_request(const struct rte_mp_msg *msg, const void *peer)
{
struct rte_mp_msg mp_resp;
const struct eal_dev_mp_req *req =
(const struct eal_dev_mp_req *)msg->param;
struct eal_dev_mp_req *resp =
(struct eal_dev_mp_req *)mp_resp.param;
struct mp_reply_bundle *bundle;
int ret = 0;
memset(&mp_resp, 0, sizeof(mp_resp));
strlcpy(mp_resp.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_resp.name));
mp_resp.len_param = sizeof(*req);
memcpy(resp, req, sizeof(*resp));
bundle = calloc(1, sizeof(*bundle));
if (bundle == NULL) {
RTE_LOG(ERR, EAL, "not enough memory\n");
resp->result = -ENOMEM;
ret = rte_mp_reply(&mp_resp, peer);
if (ret)
RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
return ret;
}
bundle->msg = *msg;
/**
* We need to send reply on interrupt thread, but peer can't be
* parsed directly, so this is a temporal hack, need to be fixed
* when it is ready.
*/
bundle->peer = (void *)strdup(peer);
if (bundle->peer == NULL) {
RTE_LOG(ERR, EAL, "not enough memory\n");
free(bundle);
resp->result = -ENOMEM;
ret = rte_mp_reply(&mp_resp, peer);
if (ret)
RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
return ret;
}
/**
* We are at IPC callback thread, sync IPC is not allowed due to
* dead lock, so we delegate the task to interrupt thread.
*/
ret = rte_eal_alarm_set(1, __handle_primary_request, bundle);
if (ret != 0) {
free(bundle->peer);
free(bundle);
resp->result = ret;
ret = rte_mp_reply(&mp_resp, peer);
if (ret != 0) {
RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
return ret;
}
}
return 0;
}
int eal_dev_hotplug_request_to_primary(struct eal_dev_mp_req *req)
{
struct rte_mp_msg mp_req;
struct rte_mp_reply mp_reply;
struct timespec ts = {.tv_sec = MP_TIMEOUT_S, .tv_nsec = 0};
struct eal_dev_mp_req *resp;
int ret;
memset(&mp_req, 0, sizeof(mp_req));
memcpy(mp_req.param, req, sizeof(*req));
mp_req.len_param = sizeof(*req);
strlcpy(mp_req.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_req.name));
ret = rte_mp_request_sync(&mp_req, &mp_reply, &ts);
if (ret || mp_reply.nb_received != 1) {
RTE_LOG(ERR, EAL, "Cannot send request to primary\n");
if (!ret)
return -1;
return ret;
}
resp = (struct eal_dev_mp_req *)mp_reply.msgs[0].param;
req->result = resp->result;
free(mp_reply.msgs);
return ret;
}
int eal_dev_hotplug_request_to_secondary(struct eal_dev_mp_req *req)
{
struct rte_mp_msg mp_req;
struct rte_mp_reply mp_reply;
struct timespec ts = {.tv_sec = MP_TIMEOUT_S, .tv_nsec = 0};
int ret;
int i;
memset(&mp_req, 0, sizeof(mp_req));
memcpy(mp_req.param, req, sizeof(*req));
mp_req.len_param = sizeof(*req);
strlcpy(mp_req.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_req.name));
ret = rte_mp_request_sync(&mp_req, &mp_reply, &ts);
if (ret != 0) {
/* if IPC is not supported, behave as if the call succeeded */
if (rte_errno != ENOTSUP)
RTE_LOG(ERR, EAL, "rte_mp_request_sync failed\n");
else
ret = 0;
return ret;
}
if (mp_reply.nb_sent != mp_reply.nb_received) {
RTE_LOG(ERR, EAL, "not all secondary reply\n");
free(mp_reply.msgs);
return -1;
}
req->result = 0;
for (i = 0; i < mp_reply.nb_received; i++) {
struct eal_dev_mp_req *resp =
(struct eal_dev_mp_req *)mp_reply.msgs[i].param;
if (resp->result != 0) {
if (req->t == EAL_DEV_REQ_TYPE_ATTACH &&
resp->result == -EEXIST)
continue;
if (req->t == EAL_DEV_REQ_TYPE_DETACH &&
resp->result == -ENOENT)
continue;
req->result = resp->result;
}
}
free(mp_reply.msgs);
return 0;
}
int eal_mp_dev_hotplug_init(void)
{
int ret;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
ret = rte_mp_action_register(EAL_DEV_MP_ACTION_REQUEST,
handle_secondary_request);
/* primary is allowed to not support IPC */
if (ret != 0 && rte_errno != ENOTSUP) {
RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
EAL_DEV_MP_ACTION_REQUEST);
return ret;
}
} else {
ret = rte_mp_action_register(EAL_DEV_MP_ACTION_REQUEST,
handle_primary_request);
if (ret != 0) {
RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
EAL_DEV_MP_ACTION_REQUEST);
return ret;
}
}
return 0;
}
void eal_mp_dev_hotplug_cleanup(void)
{
rte_mp_action_unregister(EAL_DEV_MP_ACTION_REQUEST);
}

60
Programs/hugePage/common/hotplug_mp.h Normal file
View File

@@ -0,0 +1,60 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Intel Corporation
*/
#ifndef _HOTPLUG_MP_H_
#define _HOTPLUG_MP_H_
#include "rte_dev.h"
#define EAL_DEV_MP_ACTION_REQUEST "eal_dev_mp_request"
#define EAL_DEV_MP_ACTION_RESPONSE "eal_dev_mp_response"
#define EAL_DEV_MP_DEV_NAME_MAX_LEN RTE_DEV_NAME_MAX_LEN
#define EAL_DEV_MP_BUS_NAME_MAX_LEN 32
#define EAL_DEV_MP_DEV_ARGS_MAX_LEN 128
enum eal_dev_req_type {
EAL_DEV_REQ_TYPE_ATTACH,
EAL_DEV_REQ_TYPE_DETACH,
EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK,
EAL_DEV_REQ_TYPE_DETACH_ROLLBACK,
};
struct eal_dev_mp_req {
enum eal_dev_req_type t;
char devargs[EAL_DEV_MP_DEV_ARGS_MAX_LEN];
int result;
};
/**
* Register all mp action callbacks for hotplug.
*
* @return
* 0 on success, negative on error.
*/
int
eal_mp_dev_hotplug_init(void);
/**
* Unregister all mp action callbacks for hotplug.
*/
void
eal_mp_dev_hotplug_cleanup(void);
/**
* This is a synchronous wrapper for secondary process send
* request to primary process, this is invoked when an attach
* or detach request is issued from primary process.
*/
int eal_dev_hotplug_request_to_primary(struct eal_dev_mp_req *req);
/**
* this is a synchronous wrapper for primary process send
* request to secondary process, this is invoked when an attach
* or detach request issued from secondary process.
*/
int eal_dev_hotplug_request_to_secondary(struct eal_dev_mp_req *req);
#endif /* _HOTPLUG_MP_H_ */

725
Programs/hugePage/common/malloc_elem.c Normal file
View File

@@ -0,0 +1,725 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <inttypes.h>
#include <stdint.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>
#include <sys/queue.h>
#include "rte_memory.h"
#include "rte_eal.h"
#include "rte_common.h"
#include "eal_private.h"
#include "eal_internal_cfg.h"
#include "eal_memalloc.h"
#include "malloc_elem.h"
#include "malloc_heap.h"
/*
* If debugging is enabled, freed memory is set to poison value
* to catch buggy programs. Otherwise, freed memory is set to zero
* to avoid having to zero in zmalloc
*/
#ifdef RTE_MALLOC_DEBUG
#define MALLOC_POISON 0x6b
#else
#define MALLOC_POISON 0
#endif
size_t
malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align)
{
void *cur_page, *contig_seg_start, *page_end, *cur_seg_end;
void *data_start, *data_end;
rte_iova_t expected_iova;
struct rte_memseg *ms;
size_t page_sz, cur, max;
const struct internal_config *internal_conf =
eal_get_internal_configuration();
page_sz = (size_t)elem->msl->page_sz;
data_start = RTE_PTR_ADD(elem, MALLOC_ELEM_HEADER_LEN);
data_end = RTE_PTR_ADD(elem, elem->size - MALLOC_ELEM_TRAILER_LEN);
/* segment must start after header and with specified alignment */
contig_seg_start = RTE_PTR_ALIGN_CEIL(data_start, align);
/* return if aligned address is already out of malloc element */
if (contig_seg_start > data_end)
return 0;
/* if we're in IOVA as VA mode, or if we're in legacy mode with
* hugepages, all elements are IOVA-contiguous. however, we can only
* make these assumptions about internal memory - externally allocated
* segments have to be checked.
*/
if (!elem->msl->external &&
(rte_eal_iova_mode() == RTE_IOVA_VA ||
(internal_conf->legacy_mem &&
rte_eal_has_hugepages())))
return RTE_PTR_DIFF(data_end, contig_seg_start);
cur_page = RTE_PTR_ALIGN_FLOOR(contig_seg_start, page_sz);
ms = rte_mem_virt2memseg(cur_page, elem->msl);
/* do first iteration outside the loop */
page_end = RTE_PTR_ADD(cur_page, page_sz);
cur_seg_end = RTE_MIN(page_end, data_end);
cur = RTE_PTR_DIFF(cur_seg_end, contig_seg_start) -
MALLOC_ELEM_TRAILER_LEN;
max = cur;
expected_iova = ms->iova + page_sz;
/* memsegs are contiguous in memory */
ms++;
cur_page = RTE_PTR_ADD(cur_page, page_sz);
while (cur_page < data_end) {
page_end = RTE_PTR_ADD(cur_page, page_sz);
cur_seg_end = RTE_MIN(page_end, data_end);
/* reset start of contiguous segment if unexpected iova */
if (ms->iova != expected_iova) {
/* next contiguous segment must start at specified
* alignment.
*/
contig_seg_start = RTE_PTR_ALIGN(cur_page, align);
/* new segment start may be on a different page, so find
* the page and skip to next iteration to make sure
* we're not blowing past data end.
*/
ms = rte_mem_virt2memseg(contig_seg_start, elem->msl);
cur_page = ms->addr;
/* don't trigger another recalculation */
expected_iova = ms->iova;
continue;
}
/* cur_seg_end ends on a page boundary or on data end. if we're
* looking at data end, then malloc trailer is already included
* in the calculations. if we're looking at page end, then we
* know there's more data past this page and thus there's space
* for malloc element trailer, so don't count it here.
*/
cur = RTE_PTR_DIFF(cur_seg_end, contig_seg_start);
/* update max if cur value is bigger */
if (cur > max)
max = cur;
/* move to next page */
cur_page = page_end;
expected_iova = ms->iova + page_sz;
/* memsegs are contiguous in memory */
ms++;
}
return max;
}
/*
* Initialize a general malloc_elem header structure
*/
void
malloc_elem_init(struct malloc_elem *elem, struct malloc_heap *heap,
struct rte_memseg_list *msl, size_t size,
struct malloc_elem *orig_elem, size_t orig_size, bool dirty)
{
elem->heap = heap;
elem->msl = msl;
elem->prev = NULL;
elem->next = NULL;
memset(&elem->free_list, 0, sizeof(elem->free_list));
elem->state = ELEM_FREE;
elem->dirty = dirty;
elem->size = size;
elem->pad = 0;
elem->orig_elem = orig_elem;
elem->orig_size = orig_size;
set_header(elem);
set_trailer(elem);
}
void
malloc_elem_insert(struct malloc_elem *elem)
{
struct malloc_elem *prev_elem, *next_elem;
struct malloc_heap *heap = elem->heap;
/* first and last elements must be both NULL or both non-NULL */
if ((heap->first == NULL) != (heap->last == NULL)) {
RTE_LOG(ERR, EAL, "Heap is probably corrupt\n");
return;
}
if (heap->first == NULL && heap->last == NULL) {
/* if empty heap */
heap->first = elem;
heap->last = elem;
prev_elem = NULL;
next_elem = NULL;
} else if (elem < heap->first) {
/* if lower than start */
prev_elem = NULL;
next_elem = heap->first;
heap->first = elem;
} else if (elem > heap->last) {
/* if higher than end */
prev_elem = heap->last;
next_elem = NULL;
heap->last = elem;
} else {
/* the new memory is somewhere between start and end */
uint64_t dist_from_start, dist_from_end;
dist_from_end = RTE_PTR_DIFF(heap->last, elem);
dist_from_start = RTE_PTR_DIFF(elem, heap->first);
/* check which is closer, and find closest list entries */
if (dist_from_start < dist_from_end) {
prev_elem = heap->first;
while (prev_elem->next < elem)
prev_elem = prev_elem->next;
next_elem = prev_elem->next;
} else {
next_elem = heap->last;
while (next_elem->prev > elem)
next_elem = next_elem->prev;
prev_elem = next_elem->prev;
}
}
/* insert new element */
elem->prev = prev_elem;
elem->next = next_elem;
if (prev_elem)
prev_elem->next = elem;
if (next_elem)
next_elem->prev = elem;
}
/*
* Attempt to find enough physically contiguous memory in this block to store
* our data. Assume that element has at least enough space to fit in the data,
* so we just check the page addresses.
*/
static bool
elem_check_phys_contig(const struct rte_memseg_list *msl,
void *start, size_t size)
{
return eal_memalloc_is_contig(msl, start, size);
}
/*
* calculate the starting point of where data of the requested size
* and alignment would fit in the current element. If the data doesn't
* fit, return NULL.
*/
static void *
elem_start_pt(struct malloc_elem *elem, size_t size, unsigned align,
size_t bound, bool contig)
{
size_t elem_size = elem->size;
/*
* we're allocating from the end, so adjust the size of element by
* alignment size.
*/
while (elem_size >= size) {
const size_t bmask = ~(bound - 1);
uintptr_t end_pt = (uintptr_t)elem +
elem_size - MALLOC_ELEM_TRAILER_LEN;
uintptr_t new_data_start = RTE_ALIGN_FLOOR((end_pt - size),
align);
uintptr_t new_elem_start;
/* check boundary */
if ((new_data_start & bmask) != ((end_pt - 1) & bmask)) {
end_pt = RTE_ALIGN_FLOOR(end_pt, bound);
new_data_start = RTE_ALIGN_FLOOR((end_pt - size),
align);
end_pt = new_data_start + size;
if (((end_pt - 1) & bmask) != (new_data_start & bmask))
return NULL;
}
new_elem_start = new_data_start - MALLOC_ELEM_HEADER_LEN;
/* if the new start point is before the exist start,
* it won't fit
*/
if (new_elem_start < (uintptr_t)elem)
return NULL;
if (contig) {
size_t new_data_size = end_pt - new_data_start;
/*
* if physical contiguousness was requested and we
* couldn't fit all data into one physically contiguous
* block, try again with lower addresses.
*/
if (!elem_check_phys_contig(elem->msl,
(void *)new_data_start,
new_data_size)) {
elem_size -= align;
continue;
}
}
return (void *)new_elem_start;
}
return NULL;
}
/*
* use elem_start_pt to determine if we get meet the size and
* alignment request from the current element
*/
int
malloc_elem_can_hold(struct malloc_elem *elem, size_t size, unsigned align,
size_t bound, bool contig)
{
return elem_start_pt(elem, size, align, bound, contig) != NULL;
}
/*
* split an existing element into two smaller elements at the given
* split_pt parameter.
*/
static void
split_elem(struct malloc_elem *elem, struct malloc_elem *split_pt)
{
struct malloc_elem *next_elem = elem->next;
const size_t old_elem_size = (uintptr_t)split_pt - (uintptr_t)elem;
const size_t new_elem_size = elem->size - old_elem_size;
malloc_elem_init(split_pt, elem->heap, elem->msl, new_elem_size,
elem->orig_elem, elem->orig_size, elem->dirty);
split_pt->prev = elem;
split_pt->next = next_elem;
if (next_elem)
next_elem->prev = split_pt;
else
elem->heap->last = split_pt;
elem->next = split_pt;
elem->size = old_elem_size;
set_trailer(elem);
if (elem->pad) {
/* Update inner padding inner element size. */
elem = RTE_PTR_ADD(elem, elem->pad);
elem->size = old_elem_size - elem->pad;
}
}
/*
* our malloc heap is a doubly linked list, so doubly remove our element.
*/
static void __rte_unused
remove_elem(struct malloc_elem *elem)
{
struct malloc_elem *next, *prev;
next = elem->next;
prev = elem->prev;
if (next)
next->prev = prev;
else
elem->heap->last = prev;
if (prev)
prev->next = next;
else
elem->heap->first = next;
elem->prev = NULL;
elem->next = NULL;
}
static int
next_elem_is_adjacent(struct malloc_elem *elem)
{
const struct internal_config *internal_conf =
eal_get_internal_configuration();
return elem->next == RTE_PTR_ADD(elem, elem->size) &&
elem->next->msl == elem->msl &&
(!internal_conf->match_allocations ||
elem->orig_elem == elem->next->orig_elem);
}
static int
prev_elem_is_adjacent(struct malloc_elem *elem)
{
const struct internal_config *internal_conf =
eal_get_internal_configuration();
return elem == RTE_PTR_ADD(elem->prev, elem->prev->size) &&
elem->prev->msl == elem->msl &&
(!internal_conf->match_allocations ||
elem->orig_elem == elem->prev->orig_elem);
}
/*
* Given an element size, compute its freelist index.
* We free an element into the freelist containing similarly-sized elements.
* We try to allocate elements starting with the freelist containing
* similarly-sized elements, and if necessary, we search freelists
* containing larger elements.
*
* Example element size ranges for a heap with five free lists:
* heap->free_head[0] - (0 , 2^8]
* heap->free_head[1] - (2^8 , 2^10]
* heap->free_head[2] - (2^10 ,2^12]
* heap->free_head[3] - (2^12, 2^14]
* heap->free_head[4] - (2^14, MAX_SIZE]
*/
size_t
malloc_elem_free_list_index(size_t size)
{
#define MALLOC_MINSIZE_LOG2 8
#define MALLOC_LOG2_INCREMENT 2
size_t log2;
size_t index;
if (size <= (1UL << MALLOC_MINSIZE_LOG2))
return 0;
/* Find next power of 2 >= size. */
log2 = sizeof(size) * 8 - __builtin_clzl(size - 1);
/* Compute freelist index, based on log2(size). */
index = (log2 - MALLOC_MINSIZE_LOG2 + MALLOC_LOG2_INCREMENT - 1) /
MALLOC_LOG2_INCREMENT;
return index <= RTE_HEAP_NUM_FREELISTS - 1 ?
index : RTE_HEAP_NUM_FREELISTS - 1;
}
/*
* Add the specified element to its heap's free list.
*/
void
malloc_elem_free_list_insert(struct malloc_elem *elem)
{
size_t idx;
idx = malloc_elem_free_list_index(elem->size - MALLOC_ELEM_HEADER_LEN);
elem->state = ELEM_FREE;
LIST_INSERT_HEAD(&elem->heap->free_head[idx], elem, free_list);
}
/*
* Remove the specified element from its heap's free list.
*/
void
malloc_elem_free_list_remove(struct malloc_elem *elem)
{
LIST_REMOVE(elem, free_list);
}
/*
* reserve a block of data in an existing malloc_elem. If the malloc_elem
* is much larger than the data block requested, we split the element in two.
* This function is only called from malloc_heap_alloc so parameter checking
* is not done here, as it's done there previously.
*/
struct malloc_elem *
malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align,
size_t bound, bool contig)
{
struct malloc_elem *new_elem = elem_start_pt(elem, size, align, bound,
contig);
const size_t old_elem_size = (uintptr_t)new_elem - (uintptr_t)elem;
const size_t trailer_size = elem->size - old_elem_size - size -
MALLOC_ELEM_OVERHEAD;
malloc_elem_free_list_remove(elem);
if (trailer_size > MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
/* split it, too much free space after elem */
struct malloc_elem *new_free_elem =
RTE_PTR_ADD(new_elem, size + MALLOC_ELEM_OVERHEAD);
asan_clear_split_alloczone(new_free_elem);
split_elem(elem, new_free_elem);
malloc_elem_free_list_insert(new_free_elem);
if (elem == elem->heap->last)
elem->heap->last = new_free_elem;
}
if (old_elem_size < MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
/* don't split it, pad the element instead */
elem->state = ELEM_BUSY;
elem->pad = old_elem_size;
asan_clear_alloczone(elem);
/* put a dummy header in padding, to point to real element header */
if (elem->pad > 0) { /* pad will be at least 64-bytes, as everything
* is cache-line aligned */
new_elem->pad = elem->pad;
new_elem->state = ELEM_PAD;
new_elem->size = elem->size - elem->pad;
set_header(new_elem);
}
return new_elem;
}
asan_clear_split_alloczone(new_elem);
/* we are going to split the element in two. The original element
* remains free, and the new element is the one allocated.
* Re-insert original element, in case its new size makes it
* belong on a different list.
*/
split_elem(elem, new_elem);
asan_clear_alloczone(new_elem);
new_elem->state = ELEM_BUSY;
malloc_elem_free_list_insert(elem);
return new_elem;
}
/*
* join two struct malloc_elem together. elem1 and elem2 must
* be contiguous in memory.
*/
static inline void
join_elem(struct malloc_elem *elem1, struct malloc_elem *elem2)
{
struct malloc_elem *next = elem2->next;
elem1->size += elem2->size;
if (next)
next->prev = elem1;
else
elem1->heap->last = elem1;
elem1->next = next;
elem1->dirty |= elem2->dirty;
if (elem1->pad) {
struct malloc_elem *inner = RTE_PTR_ADD(elem1, elem1->pad);
inner->size = elem1->size - elem1->pad;
}
}
struct malloc_elem *
malloc_elem_join_adjacent_free(struct malloc_elem *elem)
{
/*
* check if next element exists, is adjacent and is free, if so join
* with it, need to remove from free list.
*/
if (elem->next != NULL && elem->next->state == ELEM_FREE &&
next_elem_is_adjacent(elem)) {
void *erase;
size_t erase_len;
/* we will want to erase the trailer and header */
erase = RTE_PTR_SUB(elem->next, MALLOC_ELEM_TRAILER_LEN);
erase_len = MALLOC_ELEM_OVERHEAD + elem->next->pad;
/* remove from free list, join to this one */
malloc_elem_free_list_remove(elem->next);
join_elem(elem, elem->next);
/* erase header, trailer and pad */
memset(erase, MALLOC_POISON, erase_len);
}
/*
* check if prev element exists, is adjacent and is free, if so join
* with it, need to remove from free list.
*/
if (elem->prev != NULL && elem->prev->state == ELEM_FREE &&
prev_elem_is_adjacent(elem)) {
struct malloc_elem *new_elem;
void *erase;
size_t erase_len;
/* we will want to erase trailer and header */
erase = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
erase_len = MALLOC_ELEM_OVERHEAD + elem->pad;
/* remove from free list, join to this one */
malloc_elem_free_list_remove(elem->prev);
new_elem = elem->prev;
join_elem(new_elem, elem);
/* erase header, trailer and pad */
memset(erase, MALLOC_POISON, erase_len);
elem = new_elem;
}
return elem;
}
/*
* free a malloc_elem block by adding it to the free list. If the
* blocks either immediately before or immediately after newly freed block
* are also free, the blocks are merged together.
*/
struct malloc_elem *
malloc_elem_free(struct malloc_elem *elem)
{
void *ptr;
size_t data_len;
ptr = RTE_PTR_ADD(elem, MALLOC_ELEM_HEADER_LEN);
data_len = elem->size - MALLOC_ELEM_OVERHEAD;
/*
* Consider the element clean for the purposes of joining.
* If both neighbors are clean or non-existent,
* the joint element will be clean,
* which means the memory should be cleared.
* There is no need to clear the memory if the joint element is dirty.
*/
elem->dirty = false;
elem = malloc_elem_join_adjacent_free(elem);
malloc_elem_free_list_insert(elem);
elem->pad = 0;
/* decrease heap's count of allocated elements */
elem->heap->alloc_count--;
#ifndef RTE_MALLOC_DEBUG
/* Normally clear the memory when needed. */
if (!elem->dirty)
memset(ptr, 0, data_len);
#else
/* Always poison the memory in debug mode. */
memset(ptr, MALLOC_POISON, data_len);
#endif
return elem;
}
/* assume all checks were already done */
void
malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len)
{
struct malloc_elem *hide_start, *hide_end, *prev, *next;
size_t len_before, len_after;
hide_start = start;
hide_end = RTE_PTR_ADD(start, len);
prev = elem->prev;
next = elem->next;
/* we cannot do anything with non-adjacent elements */
if (next && next_elem_is_adjacent(elem)) {
len_after = RTE_PTR_DIFF(next, hide_end);
if (len_after >= MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
asan_clear_split_alloczone(hide_end);
/* split after */
split_elem(elem, hide_end);
malloc_elem_free_list_insert(hide_end);
} else if (len_after > 0) {
RTE_LOG(ERR, EAL, "Unaligned element, heap is probably corrupt\n");
return;
}
}
/* we cannot do anything with non-adjacent elements */
if (prev && prev_elem_is_adjacent(elem)) {
len_before = RTE_PTR_DIFF(hide_start, elem);
if (len_before >= MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
asan_clear_split_alloczone(hide_start);
/* split before */
split_elem(elem, hide_start);
prev = elem;
elem = hide_start;
malloc_elem_free_list_insert(prev);
} else if (len_before > 0) {
RTE_LOG(ERR, EAL, "Unaligned element, heap is probably corrupt\n");
return;
}
}
asan_clear_alloczone(elem);
remove_elem(elem);
}
/*
* attempt to resize a malloc_elem by expanding into any free space
* immediately after it in memory.
*/
int
malloc_elem_resize(struct malloc_elem *elem, size_t size)
{
const size_t new_size = size + elem->pad + MALLOC_ELEM_OVERHEAD;
/* if we request a smaller size, then always return ok */
if (elem->size >= new_size) {
asan_clear_alloczone(elem);
return 0;
}
/* check if there is a next element, it's free and adjacent */
if (!elem->next || elem->next->state != ELEM_FREE ||
!next_elem_is_adjacent(elem))
return -1;
if (elem->size + elem->next->size < new_size)
return -1;
/* we now know the element fits, so remove from free list,
* join the two
*/
malloc_elem_free_list_remove(elem->next);
join_elem(elem, elem->next);
if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD) {
/* now we have a big block together. Lets cut it down a bit, by splitting */
struct malloc_elem *split_pt = RTE_PTR_ADD(elem, new_size);
split_pt = RTE_PTR_ALIGN_CEIL(split_pt, RTE_CACHE_LINE_SIZE);
asan_clear_split_alloczone(split_pt);
split_elem(elem, split_pt);
malloc_elem_free_list_insert(split_pt);
}
asan_clear_alloczone(elem);
return 0;
}
static inline const char *
elem_state_to_str(enum elem_state state)
{
switch (state) {
case ELEM_PAD:
return "PAD";
case ELEM_BUSY:
return "BUSY";
case ELEM_FREE:
return "FREE";
}
return "ERROR";
}
void
malloc_elem_dump(const struct malloc_elem *elem, FILE *f)
{
fprintf(f, "Malloc element at %p (%s)\n", elem,
elem_state_to_str(elem->state));
fprintf(f, " len: 0x%zx pad: 0x%" PRIx32 "\n", elem->size, elem->pad);
fprintf(f, " prev: %p next: %p\n", elem->prev, elem->next);
}

397
Programs/hugePage/common/malloc_elem.h Normal file
View File

@@ -0,0 +1,397 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef MALLOC_ELEM_H_
#define MALLOC_ELEM_H_
#include <stdbool.h>
#include "rte_common.h"
#define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
/* dummy definition of struct so we can use pointers to it in malloc_elem struct */
struct malloc_heap;
enum elem_state {
ELEM_FREE = 0,
ELEM_BUSY,
ELEM_PAD /* element is a padding-only header */
};
struct malloc_elem {
struct malloc_heap *heap;
struct malloc_elem *volatile prev;
/**< points to prev elem in memseg */
struct malloc_elem *volatile next;
/**< points to next elem in memseg */
LIST_ENTRY(malloc_elem) free_list;
/**< list of free elements in heap */
struct rte_memseg_list *msl;
/** Element state, @c dirty and @c pad validity depends on it. */
/* An extra bit is needed to represent enum elem_state as signed int. */
enum elem_state state : 3;
/** If state == ELEM_FREE: the memory is not filled with zeroes. */
uint32_t dirty : 1;
/** Reserved for future use. */
uint32_t reserved : 28;
uint32_t pad;
size_t size;
struct malloc_elem *orig_elem;
size_t orig_size;
#ifdef RTE_MALLOC_DEBUG
uint64_t header_cookie; /* Cookie marking start of data */
/* trailer cookie at start + size */
#endif
#ifdef RTE_MALLOC_ASAN
size_t user_size;
uint64_t asan_cookie[2]; /* must be next to header_cookie */
#endif
} __rte_cache_aligned;
static const unsigned int MALLOC_ELEM_HEADER_LEN = sizeof(struct malloc_elem);
#ifndef RTE_MALLOC_DEBUG
#ifdef RTE_MALLOC_ASAN
static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
#else
static const unsigned int MALLOC_ELEM_TRAILER_LEN;
#endif
/* dummy function - just check if pointer is non-null */
static inline int
malloc_elem_cookies_ok(const struct malloc_elem *elem){ return elem != NULL; }
/* dummy function - no header if malloc_debug is not enabled */
static inline void
set_header(struct malloc_elem *elem __rte_unused){ }
/* dummy function - no trailer if malloc_debug is not enabled */
static inline void
set_trailer(struct malloc_elem *elem __rte_unused){ }
#else
static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
#define MALLOC_HEADER_COOKIE 0xbadbadbadadd2e55ULL /**< Header cookie. */
#define MALLOC_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
/* define macros to make referencing the header and trailer cookies easier */
#define MALLOC_ELEM_TRAILER(elem) (*((uint64_t*)RTE_PTR_ADD(elem, \
elem->size - MALLOC_ELEM_TRAILER_LEN)))
#define MALLOC_ELEM_HEADER(elem) (elem->header_cookie)
static inline void
set_header(struct malloc_elem *elem)
{
if (elem != NULL)
MALLOC_ELEM_HEADER(elem) = MALLOC_HEADER_COOKIE;
}
static inline void
set_trailer(struct malloc_elem *elem)
{
if (elem != NULL)
MALLOC_ELEM_TRAILER(elem) = MALLOC_TRAILER_COOKIE;
}
/* check that the header and trailer cookies are set correctly */
static inline int
malloc_elem_cookies_ok(const struct malloc_elem *elem)
{
return elem != NULL &&
MALLOC_ELEM_HEADER(elem) == MALLOC_HEADER_COOKIE &&
MALLOC_ELEM_TRAILER(elem) == MALLOC_TRAILER_COOKIE;
}
#endif
#define MALLOC_ELEM_OVERHEAD (MALLOC_ELEM_HEADER_LEN + MALLOC_ELEM_TRAILER_LEN)
#ifdef RTE_MALLOC_ASAN
/*
* ASAN_SHADOW_OFFSET should match to the corresponding
* value defined in gcc/libsanitizer/asan/asan_mapping.h
*/
#ifdef RTE_ARCH_X86_64
#define ASAN_SHADOW_OFFSET 0x00007fff8000
#elif defined(RTE_ARCH_ARM64)
#define ASAN_SHADOW_OFFSET 0x001000000000
#elif defined(RTE_ARCH_PPC_64)
#define ASAN_SHADOW_OFFSET 0x020000000000
#endif
#define ASAN_SHADOW_GRAIN_SIZE 8
#define ASAN_MEM_FREE_FLAG 0xfd
#define ASAN_MEM_REDZONE_FLAG 0xfa
#define ASAN_SHADOW_SCALE 3
#define ASAN_MEM_SHIFT(mem) ((void *)((uintptr_t)(mem) >> ASAN_SHADOW_SCALE))
#define ASAN_MEM_TO_SHADOW(mem) \
RTE_PTR_ADD(ASAN_MEM_SHIFT(mem), ASAN_SHADOW_OFFSET)
__rte_no_asan
static inline void
asan_set_shadow(void *addr, char val)
{
*(char *)addr = val;
}
static inline void
asan_set_zone(void *ptr, size_t len, uint32_t val)
{
size_t offset, i;
void *shadow;
size_t zone_len = len / ASAN_SHADOW_GRAIN_SIZE;
if (len % ASAN_SHADOW_GRAIN_SIZE != 0)
zone_len += 1;
for (i = 0; i < zone_len; i++) {
offset = i * ASAN_SHADOW_GRAIN_SIZE;
shadow = ASAN_MEM_TO_SHADOW((uintptr_t)ptr + offset);
asan_set_shadow(shadow, val);
}
}
/*
* When the memory is released, the release mark is
* set in the corresponding range of the shadow area.
*/
static inline void
asan_set_freezone(void *ptr, size_t size)
{
asan_set_zone(ptr, size, ASAN_MEM_FREE_FLAG);
}
/*
* When the memory is allocated, memory state must set as accessible.
*/
static inline void
asan_clear_alloczone(struct malloc_elem *elem)
{
asan_set_zone((void *)elem, elem->size, 0x0);
}
static inline void
asan_clear_split_alloczone(struct malloc_elem *elem)
{
void *ptr = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
asan_set_zone(ptr, MALLOC_ELEM_OVERHEAD, 0x0);
}
/*
* When the memory is allocated, the memory boundary is
* marked in the corresponding range of the shadow area.
* Requirement: redzone >= 16, is a power of two.
*/
static inline void
asan_set_redzone(struct malloc_elem *elem, size_t user_size)
{
uintptr_t head_redzone;
uintptr_t tail_redzone;
void *front_shadow;
void *tail_shadow;
uint32_t val;
if (elem != NULL) {
if (elem->state != ELEM_PAD)
elem = RTE_PTR_ADD(elem, elem->pad);
elem->user_size = user_size;
/* Set mark before the start of the allocated memory */
head_redzone = (uintptr_t)RTE_PTR_ADD(elem,
MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE);
front_shadow = ASAN_MEM_TO_SHADOW(head_redzone);
asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG);
front_shadow = ASAN_MEM_TO_SHADOW(head_redzone
- ASAN_SHADOW_GRAIN_SIZE);
asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG);
/* Set mark after the end of the allocated memory */
tail_redzone = (uintptr_t)RTE_PTR_ADD(elem,
MALLOC_ELEM_HEADER_LEN
+ elem->user_size);
tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone);
val = (tail_redzone % ASAN_SHADOW_GRAIN_SIZE);
val = (val == 0) ? ASAN_MEM_REDZONE_FLAG : val;
asan_set_shadow(tail_shadow, val);
tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone
+ ASAN_SHADOW_GRAIN_SIZE);
asan_set_shadow(tail_shadow, ASAN_MEM_REDZONE_FLAG);
}
}
/*
* When the memory is released, the mark of the memory boundary
* in the corresponding range of the shadow area is cleared.
* Requirement: redzone >= 16, is a power of two.
*/
static inline void
asan_clear_redzone(struct malloc_elem *elem)
{
uintptr_t head_redzone;
uintptr_t tail_redzone;
void *head_shadow;
void *tail_shadow;
if (elem != NULL) {
elem = RTE_PTR_ADD(elem, elem->pad);
/* Clear mark before the start of the allocated memory */
head_redzone = (uintptr_t)RTE_PTR_ADD(elem,
MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE);
head_shadow = ASAN_MEM_TO_SHADOW(head_redzone);
asan_set_shadow(head_shadow, 0x00);
head_shadow = ASAN_MEM_TO_SHADOW(head_redzone
- ASAN_SHADOW_GRAIN_SIZE);
asan_set_shadow(head_shadow, 0x00);
/* Clear mark after the end of the allocated memory */
tail_redzone = (uintptr_t)RTE_PTR_ADD(elem,
MALLOC_ELEM_HEADER_LEN + elem->user_size);
tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone);
asan_set_shadow(tail_shadow, 0x00);
tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone
+ ASAN_SHADOW_GRAIN_SIZE);
asan_set_shadow(tail_shadow, 0x00);
}
}
static inline size_t
old_malloc_size(struct malloc_elem *elem)
{
if (elem->state != ELEM_PAD)
elem = RTE_PTR_ADD(elem, elem->pad);
return elem->user_size;
}
#else /* !RTE_MALLOC_ASAN */
static inline void
asan_set_zone(void *ptr __rte_unused, size_t len __rte_unused,
uint32_t val __rte_unused) { }
static inline void
asan_set_freezone(void *ptr __rte_unused, size_t size __rte_unused) { }
static inline void
asan_clear_alloczone(struct malloc_elem *elem __rte_unused) { }
static inline void
asan_clear_split_alloczone(struct malloc_elem *elem __rte_unused) { }
static inline void
asan_set_redzone(struct malloc_elem *elem __rte_unused,
size_t user_size __rte_unused) { }
static inline void
asan_clear_redzone(struct malloc_elem *elem __rte_unused) { }
static inline size_t
old_malloc_size(struct malloc_elem *elem)
{
return elem->size - elem->pad - MALLOC_ELEM_OVERHEAD;
}
#endif /* !RTE_MALLOC_ASAN */
/*
* Given a pointer to the start of a memory block returned by malloc, get
* the actual malloc_elem header for that block.
*/
static inline struct malloc_elem *
malloc_elem_from_data(const void *data)
{
if (data == NULL)
return NULL;
struct malloc_elem *elem = RTE_PTR_SUB(data, MALLOC_ELEM_HEADER_LEN);
if (!malloc_elem_cookies_ok(elem))
return NULL;
return elem->state != ELEM_PAD ? elem: RTE_PTR_SUB(elem, elem->pad);
}
/*
* initialise a malloc_elem header
*/
void
malloc_elem_init(struct malloc_elem *elem,
struct malloc_heap *heap,
struct rte_memseg_list *msl,
size_t size,
struct malloc_elem *orig_elem,
size_t orig_size,
bool dirty);
void
malloc_elem_insert(struct malloc_elem *elem);
/*
* return true if the current malloc_elem can hold a block of data
* of the requested size and with the requested alignment
*/
int
malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
unsigned int align, size_t bound, bool contig);
/*
* reserve a block of data in an existing malloc_elem. If the malloc_elem
* is much larger than the data block requested, we split the element in two.
*/
struct malloc_elem *
malloc_elem_alloc(struct malloc_elem *elem, size_t size,
unsigned int align, size_t bound, bool contig);
/*
* free a malloc_elem block by adding it to the free list. If the
* blocks either immediately before or immediately after newly freed block
* are also free, the blocks are merged together.
*/
struct malloc_elem *
malloc_elem_free(struct malloc_elem *elem);
struct malloc_elem *
malloc_elem_join_adjacent_free(struct malloc_elem *elem);
/*
* attempt to resize a malloc_elem by expanding into any free space
* immediately after it in memory.
*/
int
malloc_elem_resize(struct malloc_elem *elem, size_t size);
void
malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len);
void
malloc_elem_free_list_remove(struct malloc_elem *elem);
/*
* dump contents of malloc elem to a file.
*/
void
malloc_elem_dump(const struct malloc_elem *elem, FILE *f);
/*
* Given an element size, compute its freelist index.
*/
size_t
malloc_elem_free_list_index(size_t size);
/*
* Add element to its heap's free list.
*/
void
malloc_elem_free_list_insert(struct malloc_elem *elem);
/*
* Find biggest IOVA-contiguous zone within an element with specified alignment.
*/
size_t
malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align);
#endif /* MALLOC_ELEM_H_ */

1447
Programs/hugePage/common/malloc_heap.c Normal file
View File

File diff suppressed because it is too large Load Diff

91
Programs/hugePage/common/malloc_heap.h Normal file
View File

@@ -0,0 +1,91 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef MALLOC_HEAP_H_
#define MALLOC_HEAP_H_
#include <stdbool.h>
#include <sys/queue.h>
#include "rte_malloc.h"
#include "rte_spinlock.h"
/* Number of free lists per heap, grouped by size. */
#define RTE_HEAP_NUM_FREELISTS 13
#define RTE_HEAP_NAME_MAX_LEN 32
/* dummy definition, for pointers */
struct malloc_elem;
/**
* Structure to hold malloc heap
*/
struct malloc_heap {
rte_spinlock_t lock;
LIST_HEAD(, malloc_elem) free_head[RTE_HEAP_NUM_FREELISTS];
struct malloc_elem *volatile first;
struct malloc_elem *volatile last;
unsigned int alloc_count;
unsigned int socket_id;
size_t total_size;
char name[RTE_HEAP_NAME_MAX_LEN];
} __rte_cache_aligned;
void *
malloc_heap_alloc(const char *type, size_t size, int socket, unsigned int flags,
size_t align, size_t bound, bool contig);
void *
malloc_heap_alloc_biggest(const char *type, int socket, unsigned int flags,
size_t align, bool contig);
int
malloc_heap_create(struct malloc_heap *heap, const char *heap_name);
int
malloc_heap_destroy(struct malloc_heap *heap);
struct rte_memseg_list *
malloc_heap_create_external_seg(void *va_addr, rte_iova_t iova_addrs[],
unsigned int n_pages, size_t page_sz, const char *seg_name,
unsigned int socket_id);
struct rte_memseg_list *
malloc_heap_find_external_seg(void *va_addr, size_t len);
int
malloc_heap_destroy_external_seg(struct rte_memseg_list *msl);
int
malloc_heap_add_external_memory(struct malloc_heap *heap,
struct rte_memseg_list *msl);
int
malloc_heap_remove_external_memory(struct malloc_heap *heap, void *va_addr,
size_t len);
int
malloc_heap_free(struct malloc_elem *elem);
int
malloc_heap_resize(struct malloc_elem *elem, size_t size);
int
malloc_heap_get_stats(struct malloc_heap *heap,
struct rte_malloc_socket_stats *socket_stats);
void
malloc_heap_dump(struct malloc_heap *heap, FILE *f);
int
malloc_socket_to_heap_id(unsigned int socket_id);
int
rte_eal_malloc_heap_init(void);
void
rte_eal_malloc_heap_cleanup(void);
#endif /* MALLOC_HEAP_H_ */

820
Programs/hugePage/common/malloc_mp.c Normal file
View File

@@ -0,0 +1,820 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Intel Corporation
*/
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include "rte_errno.h"
#include "rte_string_fns.h"
#include "eal_memalloc.h"
#include "eal_memcfg.h"
#include "eal_private.h"
#include "malloc_elem.h"
#include "malloc_mp.h"
#define MP_ACTION_SYNC "mp_malloc_sync"
/**< request sent by primary process to notify of changes in memory map */
#define MP_ACTION_ROLLBACK "mp_malloc_rollback"
/**< request sent by primary process to notify of changes in memory map. this is
* essentially a regular sync request, but we cannot send sync requests while
* another one is in progress, and we might have to - therefore, we do this as
* a separate callback.
*/
#define MP_ACTION_REQUEST "mp_malloc_request"
/**< request sent by secondary process to ask for allocation/deallocation */
#define MP_ACTION_RESPONSE "mp_malloc_response"
/**< response sent to secondary process to indicate result of request */
/* forward declarations */
static int
handle_sync_response(const struct rte_mp_msg *request,
const struct rte_mp_reply *reply);
static int
handle_rollback_response(const struct rte_mp_msg *request,
const struct rte_mp_reply *reply);
#define MP_TIMEOUT_S 5 /**< 5 seconds timeouts */
/* when we're allocating, we need to store some state to ensure that we can
* roll back later
*/
struct primary_alloc_req_state {
struct malloc_heap *heap;
struct rte_memseg **ms;
int ms_len;
struct malloc_elem *elem;
void *map_addr;
size_t map_len;
};
enum req_state {
REQ_STATE_INACTIVE = 0,
REQ_STATE_ACTIVE,
REQ_STATE_COMPLETE
};
struct mp_request {
TAILQ_ENTRY(mp_request) next;
struct malloc_mp_req user_req; /**< contents of request */
pthread_cond_t cond; /**< variable we use to time out on this request */
enum req_state state; /**< indicate status of this request */
struct primary_alloc_req_state alloc_state;
};
/*
* We could've used just a single request, but it may be possible for
* secondaries to timeout earlier than the primary, and send a new request while
* primary is still expecting replies to the old one. Therefore, each new
* request will get assigned a new ID, which is how we will distinguish between
* expected and unexpected messages.
*/
TAILQ_HEAD(mp_request_list, mp_request);
static struct {
struct mp_request_list list;
pthread_mutex_t lock;
} mp_request_list = {
.list = TAILQ_HEAD_INITIALIZER(mp_request_list.list),
.lock = PTHREAD_MUTEX_INITIALIZER
};
/**
* General workflow is the following:
*
* Allocation:
* S: send request to primary
* P: attempt to allocate memory
* if failed, sendmsg failure
* if success, send sync request
* S: if received msg of failure, quit
* if received sync request, synchronize memory map and reply with result
* P: if received sync request result
* if success, sendmsg success
* if failure, roll back allocation and send a rollback request
* S: if received msg of success, quit
* if received rollback request, synchronize memory map and reply with result
* P: if received sync request result
* sendmsg sync request result
* S: if received msg, quit
*
* Aside from timeouts, there are three points where we can quit:
* - if allocation failed straight away
* - if allocation and sync request succeeded
* - if allocation succeeded, sync request failed, allocation rolled back and
* rollback request received (irrespective of whether it succeeded or failed)
*
* Deallocation:
* S: send request to primary
* P: attempt to deallocate memory
* if failed, sendmsg failure
* if success, send sync request
* S: if received msg of failure, quit
* if received sync request, synchronize memory map and reply with result
* P: if received sync request result
* sendmsg sync request result
* S: if received msg, quit
*
* There is no "rollback" from deallocation, as it's safe to have some memory
* mapped in some processes - it's absent from the heap, so it won't get used.
*/
static struct mp_request *
find_request_by_id(uint64_t id)
{
struct mp_request *req;
TAILQ_FOREACH(req, &mp_request_list.list, next) {
if (req->user_req.id == id)
break;
}
return req;
}
/* this ID is, like, totally guaranteed to be absolutely unique. pinky swear. */
static uint64_t
get_unique_id(void)
{
uint64_t id;
do {
id = rte_rand();
} while (find_request_by_id(id) != NULL);
return id;
}
/* secondary will respond to sync requests thusly */
static int
handle_sync(const struct rte_mp_msg *msg, const void *peer)
{
struct rte_mp_msg reply;
const struct malloc_mp_req *req =
(const struct malloc_mp_req *)msg->param;
struct malloc_mp_req *resp =
(struct malloc_mp_req *)reply.param;
int ret;
if (req->t != REQ_TYPE_SYNC) {
RTE_LOG(ERR, EAL, "Unexpected request from primary\n");
return -1;
}
memset(&reply, 0, sizeof(reply));
reply.num_fds = 0;
strlcpy(reply.name, msg->name, sizeof(reply.name));
reply.len_param = sizeof(*resp);
ret = eal_memalloc_sync_with_primary();
resp->t = REQ_TYPE_SYNC;
resp->id = req->id;
resp->result = ret == 0 ? REQ_RESULT_SUCCESS : REQ_RESULT_FAIL;
return rte_mp_reply(&reply, peer);
}
static int
handle_free_request(const struct malloc_mp_req *m)
{
const struct rte_memseg_list *msl;
void *start, *end;
size_t len;
len = m->free_req.len;
start = m->free_req.addr;
end = RTE_PTR_ADD(start, len - 1);
/* check if the requested memory actually exists */
msl = rte_mem_virt2memseg_list(start);
if (msl == NULL) {
RTE_LOG(ERR, EAL, "Requested to free unknown memory\n");
return -1;
}
/* check if end is within the same memory region */
if (rte_mem_virt2memseg_list(end) != msl) {
RTE_LOG(ERR, EAL, "Requested to free memory spanning multiple regions\n");
return -1;
}
/* we're supposed to only free memory that's not external */
if (msl->external) {
RTE_LOG(ERR, EAL, "Requested to free external memory\n");
return -1;
}
/* now that we've validated the request, announce it */
eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE,
m->free_req.addr, m->free_req.len);
/* now, do the actual freeing */
return malloc_heap_free_pages(m->free_req.addr, m->free_req.len);
}
static int
handle_alloc_request(const struct malloc_mp_req *m,
struct mp_request *req)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
const struct malloc_req_alloc *ar = &m->alloc_req;
struct malloc_heap *heap;
struct malloc_elem *elem;
struct rte_memseg **ms;
size_t alloc_sz;
int n_segs;
void *map_addr;
/* this is checked by the API, but we need to prevent divide by zero */
if (ar->page_sz == 0 || !rte_is_power_of_2(ar->page_sz)) {
RTE_LOG(ERR, EAL, "Attempting to allocate with invalid page size\n");
return -1;
}
/* heap idx is index into the heap array, not socket ID */
if (ar->malloc_heap_idx >= RTE_MAX_HEAPS) {
RTE_LOG(ERR, EAL, "Attempting to allocate from invalid heap\n");
return -1;
}
heap = &mcfg->malloc_heaps[ar->malloc_heap_idx];
/*
* for allocations, we must only use internal heaps, but since the
* rte_malloc_heap_socket_is_external() is thread-safe and we're already
* read-locked, we'll have to take advantage of the fact that internal
* socket ID's are always lower than RTE_MAX_NUMA_NODES.
*/
if (heap->socket_id >= RTE_MAX_NUMA_NODES) {
RTE_LOG(ERR, EAL, "Attempting to allocate from external heap\n");
return -1;
}
alloc_sz = RTE_ALIGN_CEIL(RTE_ALIGN_CEIL(ar->elt_size, ar->align) +
MALLOC_ELEM_OVERHEAD, ar->page_sz);
n_segs = alloc_sz / ar->page_sz;
/* we can't know in advance how many pages we'll need, so we malloc */
ms = malloc(sizeof(*ms) * n_segs);
if (ms == NULL) {
RTE_LOG(ERR, EAL, "Couldn't allocate memory for request state\n");
return -1;
}
memset(ms, 0, sizeof(*ms) * n_segs);
elem = alloc_pages_on_heap(heap, ar->page_sz, ar->elt_size, ar->socket,
ar->flags, ar->align, ar->bound, ar->contig, ms,
n_segs);
if (elem == NULL)
goto fail;
map_addr = ms[0]->addr;
eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, map_addr, alloc_sz);
/* we have succeeded in allocating memory, but we still need to sync
* with other processes. however, since DPDK IPC is single-threaded, we
* send an asynchronous request and exit this callback.
*/
req->alloc_state.ms = ms;
req->alloc_state.ms_len = n_segs;
req->alloc_state.map_addr = map_addr;
req->alloc_state.map_len = alloc_sz;
req->alloc_state.elem = elem;
req->alloc_state.heap = heap;
return 0;
fail:
free(ms);
return -1;
}
/* first stage of primary handling requests from secondary */
static int
handle_request(const struct rte_mp_msg *msg, const void *peer __rte_unused)
{
const struct malloc_mp_req *m =
(const struct malloc_mp_req *)msg->param;
struct mp_request *entry;
int ret;
/* lock access to request */
pthread_mutex_lock(&mp_request_list.lock);
/* make sure it's not a dupe */
entry = find_request_by_id(m->id);
if (entry != NULL) {
RTE_LOG(ERR, EAL, "Duplicate request id\n");
goto fail;
}
entry = malloc(sizeof(*entry));
if (entry == NULL) {
RTE_LOG(ERR, EAL, "Unable to allocate memory for request\n");
goto fail;
}
/* erase all data */
memset(entry, 0, sizeof(*entry));
if (m->t == REQ_TYPE_ALLOC) {
ret = handle_alloc_request(m, entry);
} else if (m->t == REQ_TYPE_FREE) {
ret = handle_free_request(m);
} else {
RTE_LOG(ERR, EAL, "Unexpected request from secondary\n");
goto fail;
}
if (ret != 0) {
struct rte_mp_msg resp_msg;
struct malloc_mp_req *resp =
(struct malloc_mp_req *)resp_msg.param;
/* send failure message straight away */
resp_msg.num_fds = 0;
resp_msg.len_param = sizeof(*resp);
strlcpy(resp_msg.name, MP_ACTION_RESPONSE,
sizeof(resp_msg.name));
resp->t = m->t;
resp->result = REQ_RESULT_FAIL;
resp->id = m->id;
if (rte_mp_sendmsg(&resp_msg)) {
RTE_LOG(ERR, EAL, "Couldn't send response\n");
goto fail;
}
/* we did not modify the request */
free(entry);
} else {
struct rte_mp_msg sr_msg;
struct malloc_mp_req *sr =
(struct malloc_mp_req *)sr_msg.param;
struct timespec ts;
memset(&sr_msg, 0, sizeof(sr_msg));
/* we can do something, so send sync request asynchronously */
sr_msg.num_fds = 0;
sr_msg.len_param = sizeof(*sr);
strlcpy(sr_msg.name, MP_ACTION_SYNC, sizeof(sr_msg.name));
ts.tv_nsec = 0;
ts.tv_sec = MP_TIMEOUT_S;
/* sync requests carry no data */
sr->t = REQ_TYPE_SYNC;
sr->id = m->id;
/* there may be stray timeout still waiting */
do {
ret = rte_mp_request_async(&sr_msg, &ts,
handle_sync_response);
} while (ret != 0 && rte_errno == EEXIST);
if (ret != 0) {
RTE_LOG(ERR, EAL, "Couldn't send sync request\n");
if (m->t == REQ_TYPE_ALLOC)
free(entry->alloc_state.ms);
goto fail;
}
/* mark request as in progress */
memcpy(&entry->user_req, m, sizeof(*m));
entry->state = REQ_STATE_ACTIVE;
TAILQ_INSERT_TAIL(&mp_request_list.list, entry, next);
}
pthread_mutex_unlock(&mp_request_list.lock);
return 0;
fail:
pthread_mutex_unlock(&mp_request_list.lock);
free(entry);
return -1;
}
/* callback for asynchronous sync requests for primary. this will either do a
* sendmsg with results, or trigger rollback request.
*/
static int
handle_sync_response(const struct rte_mp_msg *request,
const struct rte_mp_reply *reply)
{
enum malloc_req_result result;
struct mp_request *entry;
const struct malloc_mp_req *mpreq =
(const struct malloc_mp_req *)request->param;
int i;
/* lock the request */
pthread_mutex_lock(&mp_request_list.lock);
entry = find_request_by_id(mpreq->id);
if (entry == NULL) {
RTE_LOG(ERR, EAL, "Wrong request ID\n");
goto fail;
}
result = REQ_RESULT_SUCCESS;
if (reply->nb_received != reply->nb_sent)
result = REQ_RESULT_FAIL;
for (i = 0; i < reply->nb_received; i++) {
struct malloc_mp_req *resp =
(struct malloc_mp_req *)reply->msgs[i].param;
if (resp->t != REQ_TYPE_SYNC) {
RTE_LOG(ERR, EAL, "Unexpected response to sync request\n");
result = REQ_RESULT_FAIL;
break;
}
if (resp->id != entry->user_req.id) {
RTE_LOG(ERR, EAL, "Response to wrong sync request\n");
result = REQ_RESULT_FAIL;
break;
}
if (resp->result == REQ_RESULT_FAIL) {
result = REQ_RESULT_FAIL;
break;
}
}
if (entry->user_req.t == REQ_TYPE_FREE) {
struct rte_mp_msg msg;
struct malloc_mp_req *resp = (struct malloc_mp_req *)msg.param;
memset(&msg, 0, sizeof(msg));
/* this is a free request, just sendmsg result */
resp->t = REQ_TYPE_FREE;
resp->result = result;
resp->id = entry->user_req.id;
msg.num_fds = 0;
msg.len_param = sizeof(*resp);
strlcpy(msg.name, MP_ACTION_RESPONSE, sizeof(msg.name));
if (rte_mp_sendmsg(&msg))
RTE_LOG(ERR, EAL, "Could not send message to secondary process\n");
TAILQ_REMOVE(&mp_request_list.list, entry, next);
free(entry);
} else if (entry->user_req.t == REQ_TYPE_ALLOC &&
result == REQ_RESULT_SUCCESS) {
struct malloc_heap *heap = entry->alloc_state.heap;
struct rte_mp_msg msg;
struct malloc_mp_req *resp =
(struct malloc_mp_req *)msg.param;
memset(&msg, 0, sizeof(msg));
heap->total_size += entry->alloc_state.map_len;
/* result is success, so just notify secondary about this */
resp->t = REQ_TYPE_ALLOC;
resp->result = result;
resp->id = entry->user_req.id;
msg.num_fds = 0;
msg.len_param = sizeof(*resp);
strlcpy(msg.name, MP_ACTION_RESPONSE, sizeof(msg.name));
if (rte_mp_sendmsg(&msg))
RTE_LOG(ERR, EAL, "Could not send message to secondary process\n");
TAILQ_REMOVE(&mp_request_list.list, entry, next);
free(entry->alloc_state.ms);
free(entry);
} else if (entry->user_req.t == REQ_TYPE_ALLOC &&
result == REQ_RESULT_FAIL) {
struct rte_mp_msg rb_msg;
struct malloc_mp_req *rb =
(struct malloc_mp_req *)rb_msg.param;
struct timespec ts;
struct primary_alloc_req_state *state =
&entry->alloc_state;
int ret;
memset(&rb_msg, 0, sizeof(rb_msg));
/* we've failed to sync, so do a rollback */
eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE,
state->map_addr, state->map_len);
rollback_expand_heap(state->ms, state->ms_len, state->elem,
state->map_addr, state->map_len);
/* send rollback request */
rb_msg.num_fds = 0;
rb_msg.len_param = sizeof(*rb);
strlcpy(rb_msg.name, MP_ACTION_ROLLBACK, sizeof(rb_msg.name));
ts.tv_nsec = 0;
ts.tv_sec = MP_TIMEOUT_S;
/* sync requests carry no data */
rb->t = REQ_TYPE_SYNC;
rb->id = entry->user_req.id;
/* there may be stray timeout still waiting */
do {
ret = rte_mp_request_async(&rb_msg, &ts,
handle_rollback_response);
} while (ret != 0 && rte_errno == EEXIST);
if (ret != 0) {
RTE_LOG(ERR, EAL, "Could not send rollback request to secondary process\n");
/* we couldn't send rollback request, but that's OK -
* secondary will time out, and memory has been removed
* from heap anyway.
*/
TAILQ_REMOVE(&mp_request_list.list, entry, next);
free(state->ms);
free(entry);
goto fail;
}
} else {
RTE_LOG(ERR, EAL, " to sync request of unknown type\n");
goto fail;
}
pthread_mutex_unlock(&mp_request_list.lock);
return 0;
fail:
pthread_mutex_unlock(&mp_request_list.lock);
return -1;
}
static int
handle_rollback_response(const struct rte_mp_msg *request,
const struct rte_mp_reply *reply __rte_unused)
{
struct rte_mp_msg msg;
struct malloc_mp_req *resp = (struct malloc_mp_req *)msg.param;
const struct malloc_mp_req *mpreq =
(const struct malloc_mp_req *)request->param;
struct mp_request *entry;
/* lock the request */
pthread_mutex_lock(&mp_request_list.lock);
memset(&msg, 0, sizeof(msg));
entry = find_request_by_id(mpreq->id);
if (entry == NULL) {
RTE_LOG(ERR, EAL, "Wrong request ID\n");
goto fail;
}
if (entry->user_req.t != REQ_TYPE_ALLOC) {
RTE_LOG(ERR, EAL, "Unexpected active request\n");
goto fail;
}
/* we don't care if rollback succeeded, request still failed */
resp->t = REQ_TYPE_ALLOC;
resp->result = REQ_RESULT_FAIL;
resp->id = mpreq->id;
msg.num_fds = 0;
msg.len_param = sizeof(*resp);
strlcpy(msg.name, MP_ACTION_RESPONSE, sizeof(msg.name));
if (rte_mp_sendmsg(&msg))
RTE_LOG(ERR, EAL, "Could not send message to secondary process\n");
/* clean up */
TAILQ_REMOVE(&mp_request_list.list, entry, next);
free(entry->alloc_state.ms);
free(entry);
pthread_mutex_unlock(&mp_request_list.lock);
return 0;
fail:
pthread_mutex_unlock(&mp_request_list.lock);
return -1;
}
/* final stage of the request from secondary */
static int
handle_response(const struct rte_mp_msg *msg, const void *peer __rte_unused)
{
const struct malloc_mp_req *m =
(const struct malloc_mp_req *)msg->param;
struct mp_request *entry;
pthread_mutex_lock(&mp_request_list.lock);
entry = find_request_by_id(m->id);
if (entry != NULL) {
/* update request status */
entry->user_req.result = m->result;
entry->state = REQ_STATE_COMPLETE;
/* trigger thread wakeup */
pthread_cond_signal(&entry->cond);
}
pthread_mutex_unlock(&mp_request_list.lock);
return 0;
}
/* synchronously request memory map sync, this is only called whenever primary
* process initiates the allocation.
*/
int
request_sync(void)
{
struct rte_mp_msg msg;
struct rte_mp_reply reply;
struct malloc_mp_req *req = (struct malloc_mp_req *)msg.param;
struct timespec ts;
int i, ret = -1;
memset(&msg, 0, sizeof(msg));
memset(&reply, 0, sizeof(reply));
/* no need to create tailq entries as this is entirely synchronous */
msg.num_fds = 0;
msg.len_param = sizeof(*req);
strlcpy(msg.name, MP_ACTION_SYNC, sizeof(msg.name));
/* sync request carries no data */
req->t = REQ_TYPE_SYNC;
req->id = get_unique_id();
ts.tv_nsec = 0;
ts.tv_sec = MP_TIMEOUT_S;
/* there may be stray timeout still waiting */
do {
ret = rte_mp_request_sync(&msg, &reply, &ts);
} while (ret != 0 && rte_errno == EEXIST);
if (ret != 0) {
/* if IPC is unsupported, behave as if the call succeeded */
if (rte_errno != ENOTSUP)
RTE_LOG(ERR, EAL, "Could not send sync request to secondary process\n");
else
ret = 0;
goto out;
}
if (reply.nb_received != reply.nb_sent) {
RTE_LOG(ERR, EAL, "Not all secondaries have responded\n");
goto out;
}
for (i = 0; i < reply.nb_received; i++) {
struct malloc_mp_req *resp =
(struct malloc_mp_req *)reply.msgs[i].param;
if (resp->t != REQ_TYPE_SYNC) {
RTE_LOG(ERR, EAL, "Unexpected response from secondary\n");
goto out;
}
if (resp->id != req->id) {
RTE_LOG(ERR, EAL, "Wrong request ID\n");
goto out;
}
if (resp->result != REQ_RESULT_SUCCESS) {
RTE_LOG(ERR, EAL, "Secondary process failed to synchronize\n");
goto out;
}
}
ret = 0;
out:
free(reply.msgs);
return ret;
}
/* this is a synchronous wrapper around a bunch of asynchronous requests to
* primary process. this will initiate a request and wait until responses come.
*/
int
request_to_primary(struct malloc_mp_req *user_req)
{
struct rte_mp_msg msg;
struct malloc_mp_req *msg_req = (struct malloc_mp_req *)msg.param;
struct mp_request *entry;
struct timespec ts;
struct timeval now;
int ret;
memset(&msg, 0, sizeof(msg));
memset(&ts, 0, sizeof(ts));
pthread_mutex_lock(&mp_request_list.lock);
entry = malloc(sizeof(*entry));
if (entry == NULL) {
RTE_LOG(ERR, EAL, "Cannot allocate memory for request\n");
goto fail;
}
memset(entry, 0, sizeof(*entry));
if (gettimeofday(&now, NULL) < 0) {
RTE_LOG(ERR, EAL, "Cannot get current time\n");
goto fail;
}
ts.tv_nsec = (now.tv_usec * 1000) % 1000000000;
ts.tv_sec = now.tv_sec + MP_TIMEOUT_S +
(now.tv_usec * 1000) / 1000000000;
/* initialize the request */
pthread_cond_init(&entry->cond, NULL);
msg.num_fds = 0;
msg.len_param = sizeof(*msg_req);
strlcpy(msg.name, MP_ACTION_REQUEST, sizeof(msg.name));
/* (attempt to) get a unique id */
user_req->id = get_unique_id();
/* copy contents of user request into the message */
memcpy(msg_req, user_req, sizeof(*msg_req));
if (rte_mp_sendmsg(&msg)) {
RTE_LOG(ERR, EAL, "Cannot send message to primary\n");
goto fail;
}
/* copy contents of user request into active request */
memcpy(&entry->user_req, user_req, sizeof(*user_req));
/* mark request as in progress */
entry->state = REQ_STATE_ACTIVE;
TAILQ_INSERT_TAIL(&mp_request_list.list, entry, next);
/* finally, wait on timeout */
do {
ret = pthread_cond_timedwait(&entry->cond,
&mp_request_list.lock, &ts);
} while (ret != 0 && ret != ETIMEDOUT);
if (entry->state != REQ_STATE_COMPLETE) {
RTE_LOG(ERR, EAL, "Request timed out\n");
ret = -1;
} else {
ret = 0;
user_req->result = entry->user_req.result;
}
TAILQ_REMOVE(&mp_request_list.list, entry, next);
free(entry);
pthread_mutex_unlock(&mp_request_list.lock);
return ret;
fail:
pthread_mutex_unlock(&mp_request_list.lock);
free(entry);
return -1;
}
int
register_mp_requests(void)
{
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
/* it's OK for primary to not support IPC */
if (rte_mp_action_register(MP_ACTION_REQUEST, handle_request) &&
rte_errno != ENOTSUP) {
RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
MP_ACTION_REQUEST);
return -1;
}
} else {
if (rte_mp_action_register(MP_ACTION_SYNC, handle_sync)) {
RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
MP_ACTION_SYNC);
return -1;
}
if (rte_mp_action_register(MP_ACTION_ROLLBACK, handle_sync)) {
RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
MP_ACTION_SYNC);
return -1;
}
if (rte_mp_action_register(MP_ACTION_RESPONSE,
handle_response)) {
RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
MP_ACTION_RESPONSE);
return -1;
}
}
return 0;
}
void
unregister_mp_requests(void)
{
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
rte_mp_action_unregister(MP_ACTION_REQUEST);
} else {
rte_mp_action_unregister(MP_ACTION_SYNC);
rte_mp_action_unregister(MP_ACTION_ROLLBACK);
rte_mp_action_unregister(MP_ACTION_RESPONSE);
}
}

87
Programs/hugePage/common/malloc_mp.h Normal file
View File

@@ -0,0 +1,87 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Intel Corporation
*/
#ifndef MALLOC_MP_H
#define MALLOC_MP_H
#include <stdbool.h>
#include <stdint.h>
#include "rte_common.h"
#include "rte_random.h"
/* forward declarations */
struct malloc_heap;
struct rte_memseg;
/* multiprocess synchronization structures for malloc */
enum malloc_req_type {
REQ_TYPE_ALLOC, /**< ask primary to allocate */
REQ_TYPE_FREE, /**< ask primary to free */
REQ_TYPE_SYNC /**< ask secondary to synchronize its memory map */
};
enum malloc_req_result {
REQ_RESULT_SUCCESS,
REQ_RESULT_FAIL
};
struct malloc_req_alloc {
uint32_t malloc_heap_idx;
uint64_t page_sz;
size_t elt_size;
int socket;
unsigned int flags;
size_t align;
size_t bound;
bool contig;
};
struct malloc_req_free {
RTE_STD_C11
union {
void *addr;
uint64_t addr_64;
};
size_t len;
};
struct malloc_mp_req {
enum malloc_req_type t;
RTE_STD_C11
union {
struct malloc_req_alloc alloc_req;
struct malloc_req_free free_req;
};
uint64_t id; /**< not to be populated by caller */
enum malloc_req_result result;
};
int
register_mp_requests(void);
void
unregister_mp_requests(void);
int
request_to_primary(struct malloc_mp_req *req);
/* synchronous memory map sync request */
int
request_sync(void);
/* functions from malloc_heap exposed here */
int
malloc_heap_free_pages(void *aligned_start, size_t aligned_len);
struct malloc_elem *
alloc_pages_on_heap(struct malloc_heap *heap, uint64_t pg_sz, size_t elt_size,
int socket, unsigned int flags, size_t align, size_t bound,
bool contig, struct rte_memseg **ms, int n_segs);
void
rollback_expand_heap(struct rte_memseg **ms, int n_segs,
struct malloc_elem *elem, void *map_addr, size_t map_len);
#endif /* MALLOC_MP_H */

56
Programs/hugePage/common/meson.build Normal file
View File

@@ -0,0 +1,56 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2017 Intel Corporation
includes += include_directories('.')
cflags += [ '-DABI_VERSION="@0@"'.format(abi_version) ]
sources += files(
'eal_common_bus.c',
'eal_common_class.c',
'eal_common_config.c',
'eal_common_debug.c',
'eal_common_dev.c',
'eal_common_devargs.c',
'eal_common_errno.c',
'eal_common_fbarray.c',
'eal_common_hexdump.c',
'eal_common_interrupts.c',
'eal_common_launch.c',
'eal_common_lcore.c',
'eal_common_log.c',
'eal_common_mcfg.c',
'eal_common_memalloc.c',
'eal_common_memory.c',
'eal_common_memzone.c',
'eal_common_options.c',
'eal_common_string_fns.c',
'eal_common_tailqs.c',
'eal_common_thread.c',
'eal_common_timer.c',
'eal_common_trace_points.c',
'eal_common_uuid.c',
'malloc_elem.c',
'malloc_heap.c',
'rte_malloc.c',
'rte_random.c',
'rte_reciprocal.c',
'rte_service.c',
'rte_version.c',
)
if is_linux or is_windows
sources += files('eal_common_dynmem.c')
endif
if not is_windows
sources += files(
'eal_common_cpuflags.c',
'eal_common_hypervisor.c',
'eal_common_proc.c',
'eal_common_trace.c',
'eal_common_trace_ctf.c',
'eal_common_trace_utils.c',
'hotplug_mp.c',
'malloc_mp.c',
'rte_keepalive.c',
)
endif

77
Programs/hugePage/common/rte_alarm.h Normal file
View File

@@ -0,0 +1,77 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_ALARM_H_
#define _RTE_ALARM_H_
/**
* @file
*
* Alarm functions
*
* Simple alarm-clock functionality supplied by eal.
* Does not require hpet support.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
/**
* Signature of callback back function called when an alarm goes off.
*/
typedef void (*rte_eal_alarm_callback)(void *arg);
/**
* Function to set a callback to be triggered when us microseconds
* have expired. Accuracy of timing to the microsecond is not guaranteed. The
* alarm function will not be called *before* the requested time, but may
* be called a short period of time afterwards.
* The alarm handler will be called only once. There is no need to call
* "rte_eal_alarm_cancel" from within the callback function.
*
* @param us
* The time in microseconds before the callback is called
* @param cb
* The function to be called when the alarm expires
* @param cb_arg
* Pointer parameter to be passed to the callback function
*
* @return
* On success, zero.
* On failure, a negative error number
*/
int rte_eal_alarm_set(uint64_t us, rte_eal_alarm_callback cb, void *cb_arg);
/**
* Function to cancel an alarm callback which has been registered before. If
* used outside alarm callback it wait for all callbacks to finish execution.
*
* @param cb_fn
* alarm callback
* @param cb_arg
* Pointer parameter to be passed to the callback function. To remove all
* copies of a given callback function, irrespective of parameter, (void *)-1
* can be used here.
*
* @return
* - value greater than 0 and rte_errno not changed - returned value is
* the number of canceled alarm callback functions
* - value greater or equal 0 and rte_errno set to EINPROGRESS, at least one
* alarm could not be canceled because cancellation was requested from alarm
* callback context. Returned value is the number of successfully canceled
* alarm callbacks
* - 0 and rte_errno set to ENOENT - no alarm found
* - -1 and rte_errno set to EINVAL - invalid parameter (NULL callback)
*/
int rte_eal_alarm_cancel(rte_eal_alarm_callback cb_fn, void *cb_arg);
#ifdef __cplusplus
}
#endif
#endif /* _RTE_ALARM_H_ */

598
Programs/hugePage/common/rte_bitmap.h Normal file
View File

@@ -0,0 +1,598 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef __INCLUDE_RTE_BITMAP_H__
#define __INCLUDE_RTE_BITMAP_H__
#ifdef __cplusplus
extern "C" {
#endif
/**
* @file
* RTE Bitmap
*
* The bitmap component provides a mechanism to manage large arrays of bits
* through bit get/set/clear and bit array scan operations.
*
* The bitmap scan operation is optimized for 64-bit CPUs using 64/128 byte cache
* lines. The bitmap is hierarchically organized using two arrays (array1 and
* array2), with each bit in array1 being associated with a full cache line
* (512/1024 bits) of bitmap bits, which are stored in array2: the bit in array1
* is set only when there is at least one bit set within its associated array2
* bits, otherwise the bit in array1 is cleared. The read and write operations
* for array1 and array2 are always done in slabs of 64 bits.
*
* This bitmap is not thread safe. For lock free operation on a specific bitmap
* instance, a single writer thread performing bit set/clear operations is
* allowed, only the writer thread can do bitmap scan operations, while there
* can be several reader threads performing bit get operations in parallel with
* the writer thread. When the use of locking primitives is acceptable, the
* serialization of the bit set/clear and bitmap scan operations needs to be
* enforced by the caller, while the bit get operation does not require locking
* the bitmap.
*
***/
#include <string.h>
#include "rte_compat.h"
#include "rte_common.h"
#include "rte_config.h"
#include "rte_debug.h"
#include "rte_memory.h"
#include "rte_branch_prediction.h"
#include "rte_prefetch.h"
/* Slab */
#define RTE_BITMAP_SLAB_BIT_SIZE 64
#define RTE_BITMAP_SLAB_BIT_SIZE_LOG2 6
#define RTE_BITMAP_SLAB_BIT_MASK (RTE_BITMAP_SLAB_BIT_SIZE - 1)
/* Cache line (CL) */
#define RTE_BITMAP_CL_BIT_SIZE (RTE_CACHE_LINE_SIZE * 8)
#define RTE_BITMAP_CL_BIT_SIZE_LOG2 (RTE_CACHE_LINE_SIZE_LOG2 + 3)
#define RTE_BITMAP_CL_BIT_MASK (RTE_BITMAP_CL_BIT_SIZE - 1)
#define RTE_BITMAP_CL_SLAB_SIZE (RTE_BITMAP_CL_BIT_SIZE / RTE_BITMAP_SLAB_BIT_SIZE)
#define RTE_BITMAP_CL_SLAB_SIZE_LOG2 (RTE_BITMAP_CL_BIT_SIZE_LOG2 - RTE_BITMAP_SLAB_BIT_SIZE_LOG2)
#define RTE_BITMAP_CL_SLAB_MASK (RTE_BITMAP_CL_SLAB_SIZE - 1)
/** Bitmap data structure */
struct rte_bitmap {
/* Context for array1 and array2 */
uint64_t *array1; /**< Bitmap array1 */
uint64_t *array2; /**< Bitmap array2 */
uint32_t array1_size; /**< Number of 64-bit slabs in array1 that are actually used */
uint32_t array2_size; /**< Number of 64-bit slabs in array2 */
/* Context for the "scan next" operation */
uint32_t index1; /**< Bitmap scan: Index of current array1 slab */
uint32_t offset1; /**< Bitmap scan: Offset of current bit within current array1 slab */
uint32_t index2; /**< Bitmap scan: Index of current array2 slab */
uint32_t go2; /**< Bitmap scan: Go/stop condition for current array2 cache line */
/* Storage space for array1 and array2 */
uint8_t memory[];
};
static inline void
__rte_bitmap_index1_inc(struct rte_bitmap *bmp)
{
bmp->index1 = (bmp->index1 + 1) & (bmp->array1_size - 1);
}
static inline uint64_t
__rte_bitmap_mask1_get(struct rte_bitmap *bmp)
{
return (~1llu) << bmp->offset1;
}
static inline void
__rte_bitmap_index2_set(struct rte_bitmap *bmp)
{
bmp->index2 = (((bmp->index1 << RTE_BITMAP_SLAB_BIT_SIZE_LOG2) + bmp->offset1) << RTE_BITMAP_CL_SLAB_SIZE_LOG2);
}
static inline uint32_t
__rte_bitmap_get_memory_footprint(uint32_t n_bits,
uint32_t *array1_byte_offset, uint32_t *array1_slabs,
uint32_t *array2_byte_offset, uint32_t *array2_slabs)
{
uint32_t n_slabs_context, n_slabs_array1, n_cache_lines_context_and_array1;
uint32_t n_cache_lines_array2;
uint32_t n_bytes_total;
n_cache_lines_array2 = (n_bits + RTE_BITMAP_CL_BIT_SIZE - 1) / RTE_BITMAP_CL_BIT_SIZE;
n_slabs_array1 = (n_cache_lines_array2 + RTE_BITMAP_SLAB_BIT_SIZE - 1) / RTE_BITMAP_SLAB_BIT_SIZE;
n_slabs_array1 = rte_align32pow2(n_slabs_array1);
n_slabs_context = (sizeof(struct rte_bitmap) + (RTE_BITMAP_SLAB_BIT_SIZE / 8) - 1) / (RTE_BITMAP_SLAB_BIT_SIZE / 8);
n_cache_lines_context_and_array1 = (n_slabs_context + n_slabs_array1 + RTE_BITMAP_CL_SLAB_SIZE - 1) / RTE_BITMAP_CL_SLAB_SIZE;
n_bytes_total = (n_cache_lines_context_and_array1 + n_cache_lines_array2) * RTE_CACHE_LINE_SIZE;
if (array1_byte_offset) {
*array1_byte_offset = n_slabs_context * (RTE_BITMAP_SLAB_BIT_SIZE / 8);
}
if (array1_slabs) {
*array1_slabs = n_slabs_array1;
}
if (array2_byte_offset) {
*array2_byte_offset = n_cache_lines_context_and_array1 * RTE_CACHE_LINE_SIZE;
}
if (array2_slabs) {
*array2_slabs = n_cache_lines_array2 * RTE_BITMAP_CL_SLAB_SIZE;
}
return n_bytes_total;
}
static inline void
__rte_bitmap_scan_init(struct rte_bitmap *bmp)
{
bmp->index1 = bmp->array1_size - 1;
bmp->offset1 = RTE_BITMAP_SLAB_BIT_SIZE - 1;
__rte_bitmap_index2_set(bmp);
bmp->index2 += RTE_BITMAP_CL_SLAB_SIZE;
bmp->go2 = 0;
}
/**
* Bitmap memory footprint calculation
*
* @param n_bits
* Number of bits in the bitmap
* @return
* Bitmap memory footprint measured in bytes on success, 0 on error
*/
static inline uint32_t
rte_bitmap_get_memory_footprint(uint32_t n_bits) {
/* Check input arguments */
if (n_bits == 0) {
return 0;
}
return __rte_bitmap_get_memory_footprint(n_bits, NULL, NULL, NULL, NULL);
}
/**
* Bitmap initialization
*
* @param n_bits
* Number of pre-allocated bits in array2.
* @param mem
* Base address of array1 and array2.
* @param mem_size
* Minimum expected size of bitmap.
* @return
* Handle to bitmap instance.
*/
static inline struct rte_bitmap *
rte_bitmap_init(uint32_t n_bits, uint8_t *mem, uint32_t mem_size)
{
struct rte_bitmap *bmp;
uint32_t array1_byte_offset, array1_slabs, array2_byte_offset, array2_slabs;
uint32_t size;
/* Check input arguments */
if (n_bits == 0) {
return NULL;
}
if ((mem == NULL) || (((uintptr_t) mem) & RTE_CACHE_LINE_MASK)) {
return NULL;
}
size = __rte_bitmap_get_memory_footprint(n_bits,
&array1_byte_offset, &array1_slabs,
&array2_byte_offset, &array2_slabs);
if (size > mem_size)
return NULL;
/* Setup bitmap */
memset(mem, 0, size);
bmp = (struct rte_bitmap *) mem;
bmp->array1 = (uint64_t *) &mem[array1_byte_offset];
bmp->array1_size = array1_slabs;
bmp->array2 = (uint64_t *) &mem[array2_byte_offset];
bmp->array2_size = array2_slabs;
__rte_bitmap_scan_init(bmp);
return bmp;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice.
*
* Bitmap clear slab overhead bits.
*
* @param slabs
* Slab array.
* @param slab_size
* Number of 64-bit slabs in the slabs array.
* @param pos
* The start bit position in the slabs to be cleared.
*/
__rte_experimental
static inline void
__rte_bitmap_clear_slab_overhead_bits(uint64_t *slabs, uint32_t slab_size,
uint32_t pos)
{
uint32_t i;
uint32_t index = pos / RTE_BITMAP_SLAB_BIT_SIZE;
uint32_t offset = pos & RTE_BITMAP_SLAB_BIT_MASK;
if (offset) {
for (i = offset; i < RTE_BITMAP_SLAB_BIT_SIZE; i++)
slabs[index] &= ~(1llu << i);
index++;
}
if (index < slab_size)
memset(&slabs[index], 0, sizeof(slabs[0]) *
(slab_size - index));
}
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice.
*
* Bitmap initialization with all bits set
*
* @param n_bits
* Number of pre-allocated bits in array2.
* @param mem
* Base address of array1 and array2.
* @param mem_size
* Minimum expected size of bitmap.
* @return
* Handle to bitmap instance.
*/
__rte_experimental
static inline struct rte_bitmap *
rte_bitmap_init_with_all_set(uint32_t n_bits, uint8_t *mem, uint32_t mem_size)
{
struct rte_bitmap *bmp;
uint32_t array1_byte_offset, array1_slabs;
uint32_t array2_byte_offset, array2_slabs;
uint32_t size;
/* Check input arguments */
if (!n_bits || !mem || (((uintptr_t) mem) & RTE_CACHE_LINE_MASK))
return NULL;
size = __rte_bitmap_get_memory_footprint(n_bits,
&array1_byte_offset, &array1_slabs,
&array2_byte_offset, &array2_slabs);
if (size < mem_size)
return NULL;
/* Setup bitmap */
bmp = (struct rte_bitmap *) mem;
bmp->array1 = (uint64_t *) &mem[array1_byte_offset];
bmp->array1_size = array1_slabs;
bmp->array2 = (uint64_t *) &mem[array2_byte_offset];
bmp->array2_size = array2_slabs;
__rte_bitmap_scan_init(bmp);
memset(bmp->array1, 0xff, bmp->array1_size * sizeof(bmp->array1[0]));
memset(bmp->array2, 0xff, bmp->array2_size * sizeof(bmp->array2[0]));
/* Clear overhead bits. */
__rte_bitmap_clear_slab_overhead_bits(bmp->array1, bmp->array1_size,
bmp->array2_size >> RTE_BITMAP_CL_SLAB_SIZE_LOG2);
__rte_bitmap_clear_slab_overhead_bits(bmp->array2, bmp->array2_size,
n_bits);
return bmp;
}
/**
* Bitmap free
*
* @param bmp
* Handle to bitmap instance
* @return
* 0 upon success, error code otherwise
*/
static inline int
rte_bitmap_free(struct rte_bitmap *bmp)
{
/* Check input arguments */
if (bmp == NULL) {
return -1;
}
return 0;
}
/**
* Bitmap reset
*
* @param bmp
* Handle to bitmap instance
*/
static inline void
rte_bitmap_reset(struct rte_bitmap *bmp)
{
memset(bmp->array1, 0, bmp->array1_size * sizeof(uint64_t));
memset(bmp->array2, 0, bmp->array2_size * sizeof(uint64_t));
__rte_bitmap_scan_init(bmp);
}
/**
* Bitmap location prefetch into CPU L1 cache
*
* @param bmp
* Handle to bitmap instance
* @param pos
* Bit position
*/
static inline void
rte_bitmap_prefetch0(struct rte_bitmap *bmp, uint32_t pos)
{
uint64_t *slab2;
uint32_t index2;
index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2;
slab2 = bmp->array2 + index2;
rte_prefetch0((void *) slab2);
}
/**
* Bitmap bit get
*
* @param bmp
* Handle to bitmap instance
* @param pos
* Bit position
* @return
* 0 when bit is cleared, non-zero when bit is set
*/
static inline uint64_t
rte_bitmap_get(struct rte_bitmap *bmp, uint32_t pos)
{
uint64_t *slab2;
uint32_t index2, offset2;
index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2;
offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK;
slab2 = bmp->array2 + index2;
return (*slab2) & (1llu << offset2);
}
/**
* Bitmap bit set
*
* @param bmp
* Handle to bitmap instance
* @param pos
* Bit position
*/
static inline void
rte_bitmap_set(struct rte_bitmap *bmp, uint32_t pos)
{
uint64_t *slab1, *slab2;
uint32_t index1, index2, offset1, offset2;
/* Set bit in array2 slab and set bit in array1 slab */
index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2;
offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK;
index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2);
offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK;
slab2 = bmp->array2 + index2;
slab1 = bmp->array1 + index1;
*slab2 |= 1llu << offset2;
*slab1 |= 1llu << offset1;
}
/**
* Bitmap slab set
*
* @param bmp
* Handle to bitmap instance
* @param pos
* Bit position identifying the array2 slab
* @param slab
* Value to be assigned to the 64-bit slab in array2
*/
static inline void
rte_bitmap_set_slab(struct rte_bitmap *bmp, uint32_t pos, uint64_t slab)
{
uint64_t *slab1, *slab2;
uint32_t index1, index2, offset1;
/* Set bits in array2 slab and set bit in array1 slab */
index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2;
index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2);
offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK;
slab2 = bmp->array2 + index2;
slab1 = bmp->array1 + index1;
*slab2 |= slab;
*slab1 |= 1llu << offset1;
}
#if RTE_BITMAP_CL_SLAB_SIZE == 8
static inline uint64_t
__rte_bitmap_line_not_empty(uint64_t *slab2)
{
uint64_t v1, v2, v3, v4;
v1 = slab2[0] | slab2[1];
v2 = slab2[2] | slab2[3];
v3 = slab2[4] | slab2[5];
v4 = slab2[6] | slab2[7];
v1 |= v2;
v3 |= v4;
return v1 | v3;
}
#elif RTE_BITMAP_CL_SLAB_SIZE == 16
static inline uint64_t
__rte_bitmap_line_not_empty(uint64_t *slab2)
{
uint64_t v1, v2, v3, v4, v5, v6, v7, v8;
v1 = slab2[0] | slab2[1];
v2 = slab2[2] | slab2[3];
v3 = slab2[4] | slab2[5];
v4 = slab2[6] | slab2[7];
v5 = slab2[8] | slab2[9];
v6 = slab2[10] | slab2[11];
v7 = slab2[12] | slab2[13];
v8 = slab2[14] | slab2[15];
v1 |= v2;
v3 |= v4;
v5 |= v6;
v7 |= v8;
return v1 | v3 | v5 | v7;
}
#endif /* RTE_BITMAP_CL_SLAB_SIZE */
/**
* Bitmap bit clear
*
* @param bmp
* Handle to bitmap instance
* @param pos
* Bit position
*/
static inline void
rte_bitmap_clear(struct rte_bitmap *bmp, uint32_t pos)
{
uint64_t *slab1, *slab2;
uint32_t index1, index2, offset1, offset2;
/* Clear bit in array2 slab */
index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2;
offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK;
slab2 = bmp->array2 + index2;
/* Return if array2 slab is not all-zeros */
*slab2 &= ~(1llu << offset2);
if (*slab2){
return;
}
/* Check the entire cache line of array2 for all-zeros */
index2 &= ~ RTE_BITMAP_CL_SLAB_MASK;
slab2 = bmp->array2 + index2;
if (__rte_bitmap_line_not_empty(slab2)) {
return;
}
/* The array2 cache line is all-zeros, so clear bit in array1 slab */
index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2);
offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK;
slab1 = bmp->array1 + index1;
*slab1 &= ~(1llu << offset1);
return;
}
static inline int
__rte_bitmap_scan_search(struct rte_bitmap *bmp)
{
uint64_t value1;
uint32_t i;
/* Check current array1 slab */
value1 = bmp->array1[bmp->index1];
value1 &= __rte_bitmap_mask1_get(bmp);
if (rte_bsf64_safe(value1, &bmp->offset1))
return 1;
__rte_bitmap_index1_inc(bmp);
bmp->offset1 = 0;
/* Look for another array1 slab */
for (i = 0; i < bmp->array1_size; i ++, __rte_bitmap_index1_inc(bmp)) {
value1 = bmp->array1[bmp->index1];
if (rte_bsf64_safe(value1, &bmp->offset1))
return 1;
}
return 0;
}
static inline void
__rte_bitmap_scan_read_init(struct rte_bitmap *bmp)
{
__rte_bitmap_index2_set(bmp);
bmp->go2 = 1;
rte_prefetch1((void *)(bmp->array2 + bmp->index2 + 8));
}
static inline int
__rte_bitmap_scan_read(struct rte_bitmap *bmp, uint32_t *pos, uint64_t *slab)
{
uint64_t *slab2;
slab2 = bmp->array2 + bmp->index2;
for ( ; bmp->go2 ; bmp->index2 ++, slab2 ++, bmp->go2 = bmp->index2 & RTE_BITMAP_CL_SLAB_MASK) {
if (*slab2) {
*pos = bmp->index2 << RTE_BITMAP_SLAB_BIT_SIZE_LOG2;
*slab = *slab2;
bmp->index2 ++;
slab2 ++;
bmp->go2 = bmp->index2 & RTE_BITMAP_CL_SLAB_MASK;
return 1;
}
}
return 0;
}
/**
* Bitmap scan (with automatic wrap-around)
*
* @param bmp
* Handle to bitmap instance
* @param pos
* When function call returns 1, pos contains the position of the next set
* bit, otherwise not modified
* @param slab
* When function call returns 1, slab contains the value of the entire 64-bit
* slab where the bit indicated by pos is located. Slabs are always 64-bit
* aligned, so the position of the first bit of the slab (this bit is not
* necessarily set) is pos / 64. Once a slab has been returned by the bitmap
* scan operation, the internal pointers of the bitmap are updated to point
* after this slab, so the same slab will not be returned again if it
* contains more than one bit which is set. When function call returns 0,
* slab is not modified.
* @return
* 0 if there is no bit set in the bitmap, 1 otherwise
*/
static inline int
rte_bitmap_scan(struct rte_bitmap *bmp, uint32_t *pos, uint64_t *slab)
{
/* Return data from current array2 line if available */
if (__rte_bitmap_scan_read(bmp, pos, slab)) {
return 1;
}
/* Look for non-empty array2 line */
if (__rte_bitmap_scan_search(bmp)) {
__rte_bitmap_scan_read_init(bmp);
__rte_bitmap_scan_read(bmp, pos, slab);
return 1;
}
/* Empty bitmap */
return 0;
}
#ifdef __cplusplus
}
#endif
#endif /* __INCLUDE_RTE_BITMAP_H__ */

282
Programs/hugePage/common/rte_bitops.h Normal file
View File

@@ -0,0 +1,282 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2020 Arm Limited
*/
#ifndef _RTE_BITOPS_H_
#define _RTE_BITOPS_H_
/**
* @file
* Bit Operations
*
* This file defines a family of APIs for bit operations
* without enforcing memory ordering.
*/
#include <stdint.h>
#include "rte_debug.h"
#include "rte_compat.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* Get the uint64_t value for a specified bit set.
*
* @param nr
* The bit number in range of 0 to 63.
*/
#define RTE_BIT64(nr) (UINT64_C(1) << (nr))
/**
* Get the uint32_t value for a specified bit set.
*
* @param nr
* The bit number in range of 0 to 31.
*/
#define RTE_BIT32(nr) (UINT32_C(1) << (nr))
/*------------------------ 32-bit relaxed operations ------------------------*/
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Get the target bit from a 32-bit value without memory ordering.
*
* @param nr
* The target bit to get.
* @param addr
* The address holding the bit.
* @return
* The target bit.
*/
__rte_experimental
static inline uint32_t
rte_bit_relaxed_get32(unsigned int nr, volatile uint32_t *addr)
{
RTE_ASSERT(nr < 32);
uint32_t mask = UINT32_C(1) << nr;
return (*addr) & mask;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Set the target bit in a 32-bit value to 1 without memory ordering.
*
* @param nr
* The target bit to set.
* @param addr
* The address holding the bit.
*/
__rte_experimental
static inline void
rte_bit_relaxed_set32(unsigned int nr, volatile uint32_t *addr)
{
RTE_ASSERT(nr < 32);
uint32_t mask = RTE_BIT32(nr);
*addr = (*addr) | mask;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Clear the target bit in a 32-bit value to 0 without memory ordering.
*
* @param nr
* The target bit to clear.
* @param addr
* The address holding the bit.
*/
__rte_experimental
static inline void
rte_bit_relaxed_clear32(unsigned int nr, volatile uint32_t *addr)
{
RTE_ASSERT(nr < 32);
uint32_t mask = RTE_BIT32(nr);
*addr = (*addr) & (~mask);
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Return the original bit from a 32-bit value, then set it to 1 without
* memory ordering.
*
* @param nr
* The target bit to get and set.
* @param addr
* The address holding the bit.
* @return
* The original bit.
*/
__rte_experimental
static inline uint32_t
rte_bit_relaxed_test_and_set32(unsigned int nr, volatile uint32_t *addr)
{
RTE_ASSERT(nr < 32);
uint32_t mask = RTE_BIT32(nr);
uint32_t val = *addr;
*addr = val | mask;
return val & mask;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Return the original bit from a 32-bit value, then clear it to 0 without
* memory ordering.
*
* @param nr
* The target bit to get and clear.
* @param addr
* The address holding the bit.
* @return
* The original bit.
*/
__rte_experimental
static inline uint32_t
rte_bit_relaxed_test_and_clear32(unsigned int nr, volatile uint32_t *addr)
{
RTE_ASSERT(nr < 32);
uint32_t mask = RTE_BIT32(nr);
uint32_t val = *addr;
*addr = val & (~mask);
return val & mask;
}
/*------------------------ 64-bit relaxed operations ------------------------*/
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Get the target bit from a 64-bit value without memory ordering.
*
* @param nr
* The target bit to get.
* @param addr
* The address holding the bit.
* @return
* The target bit.
*/
__rte_experimental
static inline uint64_t
rte_bit_relaxed_get64(unsigned int nr, volatile uint64_t *addr)
{
RTE_ASSERT(nr < 64);
uint64_t mask = RTE_BIT64(nr);
return (*addr) & mask;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Set the target bit in a 64-bit value to 1 without memory ordering.
*
* @param nr
* The target bit to set.
* @param addr
* The address holding the bit.
*/
__rte_experimental
static inline void
rte_bit_relaxed_set64(unsigned int nr, volatile uint64_t *addr)
{
RTE_ASSERT(nr < 64);
uint64_t mask = RTE_BIT64(nr);
(*addr) = (*addr) | mask;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Clear the target bit in a 64-bit value to 0 without memory ordering.
*
* @param nr
* The target bit to clear.
* @param addr
* The address holding the bit.
*/
__rte_experimental
static inline void
rte_bit_relaxed_clear64(unsigned int nr, volatile uint64_t *addr)
{
RTE_ASSERT(nr < 64);
uint64_t mask = RTE_BIT64(nr);
*addr = (*addr) & (~mask);
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Return the original bit from a 64-bit value, then set it to 1 without
* memory ordering.
*
* @param nr
* The target bit to get and set.
* @param addr
* The address holding the bit.
* @return
* The original bit.
*/
__rte_experimental
static inline uint64_t
rte_bit_relaxed_test_and_set64(unsigned int nr, volatile uint64_t *addr)
{
RTE_ASSERT(nr < 64);
uint64_t mask = RTE_BIT64(nr);
uint64_t val = *addr;
*addr = val | mask;
return val;
}
/**
* @warning
* @b EXPERIMENTAL: this API may change, or be removed, without prior notice
*
* Return the original bit from a 64-bit value, then clear it to 0 without
* memory ordering.
*
* @param nr
* The target bit to get and clear.
* @param addr
* The address holding the bit.
* @return
* The original bit.
*/
__rte_experimental
static inline uint64_t
rte_bit_relaxed_test_and_clear64(unsigned int nr, volatile uint64_t *addr)
{
RTE_ASSERT(nr < 64);
uint64_t mask = RTE_BIT64(nr);
uint64_t val = *addr;
*addr = val & (~mask);
return val & mask;
}
#ifdef __cplusplus
}
#endif
#endif /* _RTE_BITOPS_H_ */

49
Programs/hugePage/common/rte_branch_prediction.h Normal file
View File

@@ -0,0 +1,49 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
/**
* @file
* Branch Prediction Helpers in RTE
*/
#ifndef _RTE_BRANCH_PREDICTION_H_
#define _RTE_BRANCH_PREDICTION_H_
#ifdef __cplusplus
extern "C" {
#endif
/**
* Check if a branch is likely to be taken.
*
* This compiler builtin allows the developer to indicate if a branch is
* likely to be taken. Example:
*
* if (likely(x > 1))
* do_stuff();
*
*/
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#endif /* likely */
/**
* Check if a branch is unlikely to be taken.
*
* This compiler builtin allows the developer to indicate if a branch is
* unlikely to be taken. Example:
*
* if (unlikely(x < 1))
* do_stuff();
*
*/
#ifndef unlikely
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif /* unlikely */
#ifdef __cplusplus
}
#endif
#endif /* _RTE_BRANCH_PREDICTION_H_ */

131
Programs/hugePage/common/rte_bus.h Normal file
View File

@@ -0,0 +1,131 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2016 NXP
*/
#ifndef _RTE_BUS_H_
#define _RTE_BUS_H_
/**
* @file
*
* DPDK device bus interface
*
* This file exposes API and interfaces for bus abstraction
* over the devices and drivers in EAL.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include "rte_eal.h"
struct rte_bus;
struct rte_device;
/**
* Retrieve a bus name.
*
* @param bus
* A pointer to a rte_bus structure.
* @return
* A pointer to the bus name string.
*/
const char *rte_bus_name(const struct rte_bus *bus);
/**
* Scan all the buses.
*
* @return
* 0 in case of success in scanning all buses
* !0 in case of failure to scan
*/
int rte_bus_scan(void);
/**
* For each device on the buses, perform a driver 'match' and call the
* driver-specific probe for device initialization.
*
* @return
* 0 for successful match/probe
* !0 otherwise
*/
int rte_bus_probe(void);
/**
* Dump information of all the buses registered with EAL.
*
* @param f
* A valid and open output stream handle
*/
void rte_bus_dump(FILE *f);
/**
* Bus comparison function.
*
* @param bus
* Bus under test.
*
* @param data
* Data to compare against.
*
* @return
* 0 if the bus matches the data.
* !0 if the bus does not match.
* <0 if ordering is possible and the bus is lower than the data.
* >0 if ordering is possible and the bus is greater than the data.
*/
typedef int (*rte_bus_cmp_t)(const struct rte_bus *bus, const void *data);
/**
* Bus iterator to find a particular bus.
*
* This function compares each registered bus to find one that matches
* the data passed as parameter.
*
* If the comparison function returns zero this function will stop iterating
* over any more buses. To continue a search the bus of a previous search can
* be passed via the start parameter.
*
* @param start
* Starting point for the iteration.
*
* @param cmp
* Comparison function.
*
* @param data
* Data to pass to comparison function.
*
* @return
* A pointer to a rte_bus structure or NULL in case no bus matches
*/
struct rte_bus *rte_bus_find(const struct rte_bus *start, rte_bus_cmp_t cmp,
const void *data);
/**
* Find the registered bus for a particular device.
*/
struct rte_bus *rte_bus_find_by_device(const struct rte_device *dev);
/**
* Find the registered bus for a given name.
*/
struct rte_bus *rte_bus_find_by_name(const char *busname);
/**
* Get the common iommu class of devices bound on to buses available in the
* system. RTE_IOVA_DC means that no preference has been expressed.
*
* @return
* enum rte_iova_mode value.
*/
enum rte_iova_mode rte_bus_get_iommu_class(void);
#ifdef __cplusplus
}
#endif
#endif /* _RTE_BUS_H */

133
Programs/hugePage/common/rte_class.h Normal file
View File

@@ -0,0 +1,133 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2018 Gaëtan Rivet
*/
#ifndef _RTE_CLASS_H_
#define _RTE_CLASS_H_
/**
* @file
*
* DPDK device class interface.
*
* This file describes the interface of the device class
* abstraction layer.
*
* A device class defines the type of function a device
* will be used for e.g.: Ethernet adapter (eth),
* cryptographic co-processor (crypto), etc.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include "rte_compat.h"
#include "rte_dev.h"
/** Double linked list of classes */
RTE_TAILQ_HEAD(rte_class_list, rte_class);
/**
* A structure describing a generic device class.
*/
struct rte_class {
RTE_TAILQ_ENTRY(rte_class) next; /**< Next device class in linked list */
const char *name; /**< Name of the class */
rte_dev_iterate_t dev_iterate; /**< Device iterator. */
};
/**
* Class comparison function.
*
* @param cls
* Class under test.
*
* @param data
* Data to compare against.
*
* @return
* 0 if the class matches the data.
* !0 if the class does not match.
* <0 if ordering is possible and the class is lower than the data.
* >0 if ordering is possible and the class is greater than the data.
*/
typedef int (*rte_class_cmp_t)(const struct rte_class *cls, const void *data);
/**
* Class iterator to find a particular class.
*
* This function compares each registered class to find one that matches
* the data passed as parameter.
*
* If the comparison function returns zero this function will stop iterating
* over any more classes. To continue a search the class of a previous search
* can be passed via the start parameter.
*
* @param start
* Starting point for the iteration.
*
* @param cmp
* Comparison function.
*
* @param data
* Data to pass to comparison function.
*
* @return
* A pointer to a rte_class structure or NULL in case no class matches
*/
__rte_experimental
struct rte_class *
rte_class_find(const struct rte_class *start, rte_class_cmp_t cmp,
const void *data);
/**
* Find the registered class for a given name.
*/
__rte_experimental
struct rte_class *
rte_class_find_by_name(const char *name);
/**
* Register a Class handle.
*
* @param cls
* A pointer to a rte_class structure describing the class
* to be registered.
*/
__rte_experimental
void rte_class_register(struct rte_class *cls);
/**
* Unregister a Class handle.
*
* @param cls
* A pointer to a rte_class structure describing the class
* to be unregistered.
*/
__rte_experimental
void rte_class_unregister(struct rte_class *cls);
/**
* Helper for Class registration.
* The constructor has lower priority than Bus constructors.
* The constructor has higher priority than PMD constructors.
*/
#define RTE_REGISTER_CLASS(nm, cls) \
RTE_INIT_PRIO(classinitfn_ ##nm, CLASS) \
{\
(cls).name = RTE_STR(nm); \
rte_class_register(&cls); \
}
#define RTE_UNREGISTER_CLASS(nm, cls) \
RTE_FINI_PRIO(classfinifn_ ##nm, CLASS) \
{ \
rte_class_unregister(&cls); \
}
#ifdef __cplusplus
}
#endif
#endif /* _RTE_CLASS_H_ */

897
Programs/hugePage/common/rte_common.h Normal file
View File

@@ -0,0 +1,897 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2019 Intel Corporation
*/
#ifndef _RTE_COMMON_H_
#define _RTE_COMMON_H_
/**
* @file
*
* Generic, commonly-used macro and inline function definitions
* for DPDK.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <limits.h>
#include "rte_config.h"
/* OS specific include */
// #include "rte_os.h"
#ifndef typeof
#define typeof __typeof__
#endif
#ifndef __cplusplus
#ifndef asm
#define asm __asm__
#endif
#endif
/** C extension macro for environments lacking C11 features. */
#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112L
#define RTE_STD_C11 __extension__
#else
#define RTE_STD_C11
#endif
/*
* RTE_TOOLCHAIN_GCC is defined if the target is built with GCC,
* while a host application (like pmdinfogen) may have another compiler.
* RTE_CC_IS_GNU is true if the file is compiled with GCC,
* no matter it is a target or host application.
*/
#define RTE_CC_IS_GNU 0
#if defined __clang__
#define RTE_CC_CLANG
#elif defined __INTEL_COMPILER
#define RTE_CC_ICC
#elif defined __GNUC__
#define RTE_CC_GCC
#undef RTE_CC_IS_GNU
#define RTE_CC_IS_GNU 1
#endif
#if RTE_CC_IS_GNU
#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + \
__GNUC_PATCHLEVEL__)
#endif
/**
* Force alignment
*/
#define __rte_aligned(a) __attribute__((__aligned__(a)))
#ifdef RTE_ARCH_STRICT_ALIGN
typedef uint64_t unaligned_uint64_t __rte_aligned(1);
typedef uint32_t unaligned_uint32_t __rte_aligned(1);
typedef uint16_t unaligned_uint16_t __rte_aligned(1);
#else
typedef uint64_t unaligned_uint64_t;
typedef uint32_t unaligned_uint32_t;
typedef uint16_t unaligned_uint16_t;
#endif
/**
* Force a structure to be packed
*/
#define __rte_packed __attribute__((__packed__))
/**
* Macro to mark a type that is not subject to type-based aliasing rules
*/
#define __rte_may_alias __attribute__((__may_alias__))
/******* Macro to mark functions and fields scheduled for removal *****/
#define __rte_deprecated __attribute__((__deprecated__))
#define __rte_deprecated_msg(msg) __attribute__((__deprecated__(msg)))
/**
* Macro to mark macros and defines scheduled for removal
*/
#if defined(RTE_CC_GCC) || defined(RTE_CC_CLANG)
#define RTE_PRAGMA(x) _Pragma(#x)
#define RTE_PRAGMA_WARNING(w) RTE_PRAGMA(GCC warning #w)
#define RTE_DEPRECATED(x) RTE_PRAGMA_WARNING(#x is deprecated)
#else
#define RTE_DEPRECATED(x)
#endif
/**
* Mark a function or variable to a weak reference.
*/
#define __rte_weak __attribute__((__weak__))
/**
* Force symbol to be generated even if it appears to be unused.
*/
#define __rte_used __attribute__((used))
/*********** Macros to eliminate unused variable warnings ********/
/**
* short definition to mark a function parameter unused
*/
#define __rte_unused __attribute__((__unused__))
/**
* Mark pointer as restricted with regard to pointer aliasing.
*/
#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#define __rte_restrict __restrict
#else
#define __rte_restrict restrict
#endif
/**
* definition to mark a variable or function parameter as used so
* as to avoid a compiler warning
*/
#define RTE_SET_USED(x) (void)(x)
/**
* Check format string and its arguments at compile-time.
*
* GCC on Windows assumes MS-specific format string by default,
* even if the underlying stdio implementation is ANSI-compliant,
* so this must be overridden.
*/
#if RTE_CC_IS_GNU
#define __rte_format_printf(format_index, first_arg) \
__attribute__((format(gnu_printf, format_index, first_arg)))
#else
#define __rte_format_printf(format_index, first_arg) \
__attribute__((format(printf, format_index, first_arg)))
#endif
/**
* Tells compiler that the function returns a value that points to
* memory, where the size is given by the one or two arguments.
* Used by compiler to validate object size.
*/
#if defined(RTE_CC_GCC) || defined(RTE_CC_CLANG)
#define __rte_alloc_size(...) \
__attribute__((alloc_size(__VA_ARGS__)))
#else
#define __rte_alloc_size(...)
#endif
#define RTE_PRIORITY_LOG 101
#define RTE_PRIORITY_BUS 110
#define RTE_PRIORITY_CLASS 120
#define RTE_PRIORITY_LAST 65535
#define RTE_PRIO(prio) \
RTE_PRIORITY_ ## prio
/**
* Run function before main() with high priority.
*
* @param func
* Constructor function.
* @param prio
* Priority number must be above 100.
* Lowest number is the first to run.
*/
#ifndef RTE_INIT_PRIO /* Allow to override from EAL */
#define RTE_INIT_PRIO(func, prio) \
static void __attribute__((constructor(RTE_PRIO(prio)), used)) func(void)
#endif
/**
* Run function before main() with low priority.
*
* The constructor will be run after prioritized constructors.
*
* @param func
* Constructor function.
*/
#define RTE_INIT(func) \
RTE_INIT_PRIO(func, LAST)
/**
* Run after main() with low priority.
*
* @param func
* Destructor function name.
* @param prio
* Priority number must be above 100.
* Lowest number is the last to run.
*/
#ifndef RTE_FINI_PRIO /* Allow to override from EAL */
#define RTE_FINI_PRIO(func, prio) \
static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
#endif
/**
* Run after main() with high priority.
*
* The destructor will be run *before* prioritized destructors.
*
* @param func
* Destructor function name.
*/
#define RTE_FINI(func) \
RTE_FINI_PRIO(func, LAST)
/**
* Hint never returning function
*/
#define __rte_noreturn __attribute__((noreturn))
/**
* Issue a warning in case the function's return value is ignored.
*
* The use of this attribute should be restricted to cases where
* ignoring the marked function's return value is almost always a
* bug. With GCC, some effort is required to make clear that ignoring
* the return value is intentional. The usual void-casting method to
* mark something unused as used does not suppress the warning with
* this compiler.
*
* @code{.c}
* __rte_warn_unused_result int foo();
*
* void ignore_foo_result(void) {
* foo(); // generates a warning with all compilers
*
* (void)foo(); // still generates the warning with GCC (but not clang)
*
* int unused __rte_unused;
* unused = foo(); // does the trick with all compilers
* }
* @endcode
*/
#define __rte_warn_unused_result __attribute__((warn_unused_result))
/**
* Force a function to be inlined
*/
#define __rte_always_inline inline __attribute__((always_inline))
/**
* Force a function to be noinlined
*/
#define __rte_noinline __attribute__((noinline))
/**
* Hint function in the hot path
*/
#define __rte_hot __attribute__((hot))
/**
* Hint function in the cold path
*/
#define __rte_cold __attribute__((cold))
/**
* Disable AddressSanitizer on some code
*/
#ifdef RTE_MALLOC_ASAN
#ifdef RTE_CC_CLANG
#define __rte_no_asan __attribute__((no_sanitize("address", "hwaddress")))
#else
#define __rte_no_asan __attribute__((no_sanitize_address))
#endif
#else /* ! RTE_MALLOC_ASAN */
#define __rte_no_asan
#endif
/*********** Macros for pointer arithmetic ********/
/**
* add a byte-value offset to a pointer
*/
#define RTE_PTR_ADD(ptr, x) ((void*)((uintptr_t)(ptr) + (x)))
/**
* subtract a byte-value offset from a pointer
*/
#define RTE_PTR_SUB(ptr, x) ((void *)((uintptr_t)(ptr) - (x)))
/**
* get the difference between two pointer values, i.e. how far apart
* in bytes are the locations they point two. It is assumed that
* ptr1 is greater than ptr2.
*/
#define RTE_PTR_DIFF(ptr1, ptr2) ((uintptr_t)(ptr1) - (uintptr_t)(ptr2))
/**
* Workaround to cast a const field of a structure to non-const type.
*/
#define RTE_CAST_FIELD(var, field, type) \
(*(type *)((uintptr_t)(var) + offsetof(typeof(*(var)), field)))
/*********** Macros/static functions for doing alignment ********/
/**
* Macro to align a pointer to a given power-of-two. The resultant
* pointer will be a pointer of the same type as the first parameter, and
* point to an address no higher than the first parameter. Second parameter
* must be a power-of-two value.
*/
#define RTE_PTR_ALIGN_FLOOR(ptr, align) \
((typeof(ptr))RTE_ALIGN_FLOOR((uintptr_t)(ptr), align))
/**
* Macro to align a value to a given power-of-two. The resultant value
* will be of the same type as the first parameter, and will be no
* bigger than the first parameter. Second parameter must be a
* power-of-two value.
*/
#define RTE_ALIGN_FLOOR(val, align) \
(typeof(val))((val) & (~((typeof(val))((align) - 1))))
/**
* Macro to align a pointer to a given power-of-two. The resultant
* pointer will be a pointer of the same type as the first parameter, and
* point to an address no lower than the first parameter. Second parameter
* must be a power-of-two value.
*/
#define RTE_PTR_ALIGN_CEIL(ptr, align) \
RTE_PTR_ALIGN_FLOOR((typeof(ptr))RTE_PTR_ADD(ptr, (align) - 1), align)
/**
* Macro to align a value to a given power-of-two. The resultant value
* will be of the same type as the first parameter, and will be no lower
* than the first parameter. Second parameter must be a power-of-two
* value.
*/
#define RTE_ALIGN_CEIL(val, align) \
RTE_ALIGN_FLOOR(((val) + ((typeof(val)) (align) - 1)), align)
/**
* Macro to align a pointer to a given power-of-two. The resultant
* pointer will be a pointer of the same type as the first parameter, and
* point to an address no lower than the first parameter. Second parameter
* must be a power-of-two value.
* This function is the same as RTE_PTR_ALIGN_CEIL
*/
#define RTE_PTR_ALIGN(ptr, align) RTE_PTR_ALIGN_CEIL(ptr, align)
/**
* Macro to align a value to a given power-of-two. The resultant
* value will be of the same type as the first parameter, and
* will be no lower than the first parameter. Second parameter
* must be a power-of-two value.
* This function is the same as RTE_ALIGN_CEIL
*/
#define RTE_ALIGN(val, align) RTE_ALIGN_CEIL(val, align)
/**
* Macro to align a value to the multiple of given value. The resultant
* value will be of the same type as the first parameter and will be no lower
* than the first parameter.
*/
#define RTE_ALIGN_MUL_CEIL(v, mul) \
((((v) + (typeof(v))(mul) - 1) / ((typeof(v))(mul))) * (typeof(v))(mul))
/**
* Macro to align a value to the multiple of given value. The resultant
* value will be of the same type as the first parameter and will be no higher
* than the first parameter.
*/
#define RTE_ALIGN_MUL_FLOOR(v, mul) \
(((v) / ((typeof(v))(mul))) * (typeof(v))(mul))
/**
* Macro to align value to the nearest multiple of the given value.
* The resultant value might be greater than or less than the first parameter
* whichever difference is the lowest.
*/
#define RTE_ALIGN_MUL_NEAR(v, mul) \
({ \
typeof(v) ceil = RTE_ALIGN_MUL_CEIL(v, mul); \
typeof(v) floor = RTE_ALIGN_MUL_FLOOR(v, mul); \
(ceil - (v)) > ((v) - floor) ? floor : ceil; \
})
/**
* Checks if a pointer is aligned to a given power-of-two value
*
* @param ptr
* The pointer whose alignment is to be checked
* @param align
* The power-of-two value to which the ptr should be aligned
*
* @return
* True(1) where the pointer is correctly aligned, false(0) otherwise
*/
static inline int
rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
{
return ((uintptr_t)ptr & (align - 1)) == 0;
}
/*********** Macros for compile type checks ********/
/**
* Triggers an error at compilation time if the condition is true.
*/
#define RTE_BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
/*********** Cache line related macros ********/
/** Cache line mask. */
#define RTE_CACHE_LINE_MASK (RTE_CACHE_LINE_SIZE-1)
/** Return the first cache-aligned value greater or equal to size. */
#define RTE_CACHE_LINE_ROUNDUP(size) RTE_ALIGN_CEIL(size, RTE_CACHE_LINE_SIZE)
/** Cache line size in terms of log2 */
// #if RTE_CACHE_LINE_SIZE == 64
// #define RTE_CACHE_LINE_SIZE_LOG2 6
// #elif RTE_CACHE_LINE_SIZE == 128
// #define RTE_CACHE_LINE_SIZE_LOG2 7
// #else
// #error "Unsupported cache line size"
// #endif
/** Minimum Cache line size. */
#define RTE_CACHE_LINE_MIN_SIZE 64
/** Force alignment to cache line. */
#define __rte_cache_aligned __rte_aligned(RTE_CACHE_LINE_SIZE)
/** Force minimum cache line alignment. */
#define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
/*********** PA/IOVA type definitions ********/
/** Physical address */
typedef uint64_t phys_addr_t;
#define RTE_BAD_PHYS_ADDR ((phys_addr_t)-1)
/**
* IO virtual address type.
* When the physical addressing mode (IOVA as PA) is in use,
* the translation from an IO virtual address (IOVA) to a physical address
* is a direct mapping, i.e. the same value.
* Otherwise, in virtual mode (IOVA as VA), an IOMMU may do the translation.
*/
typedef uint64_t rte_iova_t;
#define RTE_BAD_IOVA ((rte_iova_t)-1)
/*********** Structure alignment markers ********/
/** Generic marker for any place in a structure. */
__extension__ typedef void *RTE_MARKER[0];
/** Marker for 1B alignment in a structure. */
__extension__ typedef uint8_t RTE_MARKER8[0];
/** Marker for 2B alignment in a structure. */
__extension__ typedef uint16_t RTE_MARKER16[0];
/** Marker for 4B alignment in a structure. */
__extension__ typedef uint32_t RTE_MARKER32[0];
/** Marker for 8B alignment in a structure. */
__extension__ typedef uint64_t RTE_MARKER64[0];
/**
* Combines 32b inputs most significant set bits into the least
* significant bits to construct a value with the same MSBs as x
* but all 1's under it.
*
* @param x
* The integer whose MSBs need to be combined with its LSBs
* @return
* The combined value.
*/
static inline uint32_t
rte_combine32ms1b(uint32_t x)
{
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
return x;
}
/**
* Combines 64b inputs most significant set bits into the least
* significant bits to construct a value with the same MSBs as x
* but all 1's under it.
*
* @param v
* The integer whose MSBs need to be combined with its LSBs
* @return
* The combined value.
*/
static inline uint64_t
rte_combine64ms1b(uint64_t v)
{
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
v |= v >> 32;
return v;
}
/*********** Macros to work with powers of 2 ********/
/**
* Macro to return 1 if n is a power of 2, 0 otherwise
*/
#define RTE_IS_POWER_OF_2(n) ((n) && !(((n) - 1) & (n)))
/**
* Returns true if n is a power of 2
* @param n
* Number to check
* @return 1 if true, 0 otherwise
*/
static inline int
rte_is_power_of_2(uint32_t n)
{
return n && !(n & (n - 1));
}
/**
* Aligns input parameter to the next power of 2
*
* @param x
* The integer value to align
*
* @return
* Input parameter aligned to the next power of 2
*/
static inline uint32_t
rte_align32pow2(uint32_t x)
{
x--;
x = rte_combine32ms1b(x);
return x + 1;
}
/**
* Aligns input parameter to the previous power of 2
*
* @param x
* The integer value to align
*
* @return
* Input parameter aligned to the previous power of 2
*/
static inline uint32_t
rte_align32prevpow2(uint32_t x)
{
x = rte_combine32ms1b(x);
return x - (x >> 1);
}
/**
* Aligns 64b input parameter to the next power of 2
*
* @param v
* The 64b value to align
*
* @return
* Input parameter aligned to the next power of 2
*/
static inline uint64_t
rte_align64pow2(uint64_t v)
{
v--;
v = rte_combine64ms1b(v);
return v + 1;
}
/**
* Aligns 64b input parameter to the previous power of 2
*
* @param v
* The 64b value to align
*
* @return
* Input parameter aligned to the previous power of 2
*/
static inline uint64_t
rte_align64prevpow2(uint64_t v)
{
v = rte_combine64ms1b(v);
return v - (v >> 1);
}
/*********** Macros for calculating min and max **********/
/**
* Macro to return the minimum of two numbers
*/
#define RTE_MIN(a, b) \
__extension__ ({ \
typeof (a) _a = (a); \
typeof (b) _b = (b); \
_a < _b ? _a : _b; \
})
/**
* Macro to return the maximum of two numbers
*/
#define RTE_MAX(a, b) \
__extension__ ({ \
typeof (a) _a = (a); \
typeof (b) _b = (b); \
_a > _b ? _a : _b; \
})
/*********** Other general functions / macros ********/
/**
* Searches the input parameter for the least significant set bit
* (starting from zero).
* If a least significant 1 bit is found, its bit index is returned.
* If the content of the input parameter is zero, then the content of the return
* value is undefined.
* @param v
* input parameter, should not be zero.
* @return
* least significant set bit in the input parameter.
*/
static inline uint32_t
rte_bsf32(uint32_t v)
{
return (uint32_t)__builtin_ctz(v);
}
/**
* Searches the input parameter for the least significant set bit
* (starting from zero). Safe version (checks for input parameter being zero).
*
* @warning ``pos`` must be a valid pointer. It is not checked!
*
* @param v
* The input parameter.
* @param pos
* If ``v`` was not 0, this value will contain position of least significant
* bit within the input parameter.
* @return
* Returns 0 if ``v`` was 0, otherwise returns 1.
*/
static inline int
rte_bsf32_safe(uint32_t v, uint32_t *pos)
{
if (v == 0)
return 0;
*pos = rte_bsf32(v);
return 1;
}
/**
* Return the rounded-up log2 of a integer.
*
* @note Contrary to the logarithm mathematical operation,
* rte_log2_u32(0) == 0 and not -inf.
*
* @param v
* The input parameter.
* @return
* The rounded-up log2 of the input, or 0 if the input is 0.
*/
static inline uint32_t
rte_log2_u32(uint32_t v)
{
if (v == 0)
return 0;
v = rte_align32pow2(v);
return rte_bsf32(v);
}
/**
* Return the last (most-significant) bit set.
*
* @note The last (most significant) bit is at position 32.
* @note rte_fls_u32(0) = 0, rte_fls_u32(1) = 1, rte_fls_u32(0x80000000) = 32
*
* @param x
* The input parameter.
* @return
* The last (most-significant) bit set, or 0 if the input is 0.
*/
static inline uint32_t
rte_fls_u32(uint32_t x)
{
return (x == 0) ? 0 : 32 - __builtin_clz(x);
}
/**
* Searches the input parameter for the least significant set bit
* (starting from zero).
* If a least significant 1 bit is found, its bit index is returned.
* If the content of the input parameter is zero, then the content of the return
* value is undefined.
* @param v
* input parameter, should not be zero.
* @return
* least significant set bit in the input parameter.
*/
static inline uint32_t
rte_bsf64(uint64_t v)
{
return (uint32_t)__builtin_ctzll(v);
}
/**
* Searches the input parameter for the least significant set bit
* (starting from zero). Safe version (checks for input parameter being zero).
*
* @warning ``pos`` must be a valid pointer. It is not checked!
*
* @param v
* The input parameter.
* @param pos
* If ``v`` was not 0, this value will contain position of least significant
* bit within the input parameter.
* @return
* Returns 0 if ``v`` was 0, otherwise returns 1.
*/
static inline int
rte_bsf64_safe(uint64_t v, uint32_t *pos)
{
if (v == 0)
return 0;
*pos = rte_bsf64(v);
return 1;
}
/**
* Return the last (most-significant) bit set.
*
* @note The last (most significant) bit is at position 64.
* @note rte_fls_u64(0) = 0, rte_fls_u64(1) = 1,
* rte_fls_u64(0x8000000000000000) = 64
*
* @param x
* The input parameter.
* @return
* The last (most-significant) bit set, or 0 if the input is 0.
*/
static inline uint32_t
rte_fls_u64(uint64_t x)
{
return (x == 0) ? 0 : 64 - __builtin_clzll(x);
}
/**
* Return the rounded-up log2 of a 64-bit integer.
*
* @note Contrary to the logarithm mathematical operation,
* rte_log2_u64(0) == 0 and not -inf.
*
* @param v
* The input parameter.
* @return
* The rounded-up log2 of the input, or 0 if the input is 0.
*/
static inline uint32_t
rte_log2_u64(uint64_t v)
{
if (v == 0)
return 0;
v = rte_align64pow2(v);
/* we checked for v being 0 already, so no undefined behavior */
return rte_bsf64(v);
}
#ifndef offsetof
/** Return the offset of a field in a structure. */
#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
#endif
/**
* Return pointer to the wrapping struct instance.
*
* Example:
*
* struct wrapper {
* ...
* struct child c;
* ...
* };
*
* struct child *x = obtain(...);
* struct wrapper *w = container_of(x, struct wrapper, c);
*/
#ifndef container_of
#define container_of(ptr, type, member) __extension__ ({ \
const typeof(((type *)0)->member) *_ptr = (ptr); \
__rte_unused type *_target_ptr = \
(type *)(ptr); \
(type *)(((uintptr_t)_ptr) - offsetof(type, member)); \
})
#endif
/** Swap two variables. */
#define RTE_SWAP(a, b) \
__extension__ ({ \
typeof (a) _a = a; \
a = b; \
b = _a; \
})
/**
* Get the size of a field in a structure.
*
* @param type
* The type of the structure.
* @param field
* The field in the structure.
* @return
* The size of the field in the structure, in bytes.
*/
#define RTE_SIZEOF_FIELD(type, field) (sizeof(((type *)0)->field))
#define _RTE_STR(x) #x
/** Take a macro value and get a string version of it */
#define RTE_STR(x) _RTE_STR(x)
/**
* ISO C helpers to modify format strings using variadic macros.
* This is a replacement for the ", ## __VA_ARGS__" GNU extension.
* An empty %s argument is appended to avoid a dangling comma.
*/
#define RTE_FMT(fmt, ...) fmt "%.0s", __VA_ARGS__ ""
#define RTE_FMT_HEAD(fmt, ...) fmt
#define RTE_FMT_TAIL(fmt, ...) __VA_ARGS__
/** Mask value of type "tp" for the first "ln" bit set. */
#define RTE_LEN2MASK(ln, tp) \
((tp)((uint64_t)-1 >> (sizeof(uint64_t) * CHAR_BIT - (ln))))
/** Number of elements in the array. */
#define RTE_DIM(a) (sizeof (a) / sizeof ((a)[0]))
/**
* Converts a numeric string to the equivalent uint64_t value.
* As well as straight number conversion, also recognises the suffixes
* k, m and g for kilobytes, megabytes and gigabytes respectively.
*
* If a negative number is passed in i.e. a string with the first non-black
* character being "-", zero is returned. Zero is also returned in the case of
* an error with the strtoull call in the function.
*
* @param str
* String containing number to convert.
* @return
* Number.
*/
uint64_t
rte_str_to_size(const char *str);
/**
* Function to terminate the application immediately, printing an error
* message and returning the exit_code back to the shell.
*
* This function never returns
*
* @param exit_code
* The exit code to be returned by the application
* @param format
* The format string to be used for printing the message. This can include
* printf format characters which will be expanded using any further parameters
* to the function.
*/
__rte_noreturn void
rte_exit(int exit_code, const char *format, ...)
__rte_format_printf(2, 3);
#ifdef __cplusplus
}
#endif
#endif

57
Programs/hugePage/common/rte_compat.h Normal file
View File

@@ -0,0 +1,57 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2015 Neil Horman <nhorman@tuxdriver.com>.
* All rights reserved.
*/
#ifndef _RTE_COMPAT_H_
#define _RTE_COMPAT_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifndef ALLOW_EXPERIMENTAL_API
#define __rte_experimental \
__attribute__((deprecated("Symbol is not yet part of stable ABI"), \
section(".text.experimental")))
#else
#define __rte_experimental \
__attribute__((section(".text.experimental")))
#endif
#ifndef __has_attribute
/* if no has_attribute assume no support for attribute too */
#define __has_attribute(x) 0
#endif
#if !defined ALLOW_INTERNAL_API && __has_attribute(error) /* For GCC */
#define __rte_internal \
__attribute__((error("Symbol is not public ABI"), \
section(".text.internal")))
#elif !defined ALLOW_INTERNAL_API && __has_attribute(diagnose_if) /* For clang */
#define __rte_internal \
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wgcc-compat\"") \
__attribute__((diagnose_if(1, "Symbol is not public ABI", "error"), \
section(".text.internal"))) \
_Pragma("GCC diagnostic pop")
#else
#define __rte_internal \
__attribute__((section(".text.internal")))
#endif
#ifdef __cplusplus
}
#endif
#endif /* _RTE_COMPAT_H_ */

136
Programs/hugePage/common/rte_config.h Normal file
View File

@@ -0,0 +1,136 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017 Intel Corporation
*/
/**
* @file Header file containing DPDK compilation parameters
*
* Header file containing DPDK compilation parameters. Also include the
* meson-generated header file containing the detected parameters that
* are variable across builds or build environments.
*/
#ifndef _RTE_CONFIG_H_
#define _RTE_CONFIG_H_
// #include "rte_build_config.h"
/* legacy defines */
#ifdef RTE_EXEC_ENV_LINUX
#define RTE_EXEC_ENV_LINUXAPP 1
#endif
#ifdef RTE_EXEC_ENV_FREEBSD
#define RTE_EXEC_ENV_BSDAPP 1
#endif
/* String that appears before the version number */
#define RTE_VER_PREFIX "DPDK"
/****** library defines ********/
/* EAL defines */
#define RTE_MAX_HEAPS 32
#define RTE_MAX_MEMSEG_LISTS 128
#define RTE_MAX_MEMSEG_PER_LIST 8192
#define RTE_MAX_MEM_MB_PER_LIST 32768
#define RTE_MAX_MEMSEG_PER_TYPE 32768
#define RTE_MAX_MEM_MB_PER_TYPE 65536
#define RTE_MAX_MEMZONE 2560
#define RTE_MAX_TAILQ 32
#define RTE_LOG_DP_LEVEL RTE_LOG_INFO
#define RTE_MAX_VFIO_CONTAINERS 64
/* bsd module defines */
#define RTE_CONTIGMEM_MAX_NUM_BUFS 64
#define RTE_CONTIGMEM_DEFAULT_NUM_BUFS 1
#define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
/* mempool defines */
#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
/* RTE_LIBRTE_MEMPOOL_STATS is not set */
/* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
/* mbuf defines */
#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
#define RTE_PKTMBUF_HEADROOM 128
/* ether defines */
#define RTE_MAX_QUEUES_PER_PORT 1024
#define RTE_ETHDEV_QUEUE_STAT_CNTRS 16 /* max 256 */
#define RTE_ETHDEV_RXTX_CALLBACKS 1
#define RTE_MAX_MULTI_HOST_CTRLS 4
/* cryptodev defines */
#define RTE_CRYPTO_MAX_DEVS 64
#define RTE_CRYPTODEV_NAME_LEN 64
#define RTE_CRYPTO_CALLBACKS 1
/* compressdev defines */
#define RTE_COMPRESS_MAX_DEVS 64
/* regexdev defines */
#define RTE_MAX_REGEXDEV_DEVS 32
/* eventdev defines */
#define RTE_EVENT_MAX_DEVS 16
#define RTE_EVENT_MAX_PORTS_PER_DEV 255
#define RTE_EVENT_MAX_QUEUES_PER_DEV 255
#define RTE_EVENT_TIMER_ADAPTER_NUM_MAX 32
#define RTE_EVENT_ETH_INTR_RING_SIZE 1024
#define RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE 32
#define RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE 32
/* rawdev defines */
#define RTE_RAWDEV_MAX_DEVS 64
/* ip_fragmentation defines */
#define RTE_LIBRTE_IP_FRAG_MAX_FRAG 8
// RTE_LIBRTE_IP_FRAG_TBL_STAT is not set
/* rte_power defines */
#define RTE_MAX_LCORE_FREQS 64
/* rte_graph defines */
#define RTE_GRAPH_BURST_SIZE 256
#define RTE_LIBRTE_GRAPH_STATS 1
/****** driver defines ********/
/* Packet prefetching in PMDs */
#define RTE_PMD_PACKET_PREFETCH 1
/* QuickAssist device */
/* Max. number of QuickAssist devices which can be attached */
#define RTE_PMD_QAT_MAX_PCI_DEVICES 48
#define RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS 16
#define RTE_PMD_QAT_COMP_IM_BUFFER_SIZE 65536
/* virtio crypto defines */
#define RTE_MAX_VIRTIO_CRYPTO 32
/* DPAA SEC max cryptodev devices*/
#define RTE_LIBRTE_DPAA_MAX_CRYPTODEV 4
/* fm10k defines */
#define RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE 1
/* hns3 defines */
#define RTE_LIBRTE_HNS3_MAX_TQP_NUM_PER_PF 256
/* i40e defines */
#define RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 1
// RTE_LIBRTE_I40E_16BYTE_RX_DESC is not set
#define RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF 64
#define RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF 4
#define RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM 4
/* Ring net PMD settings */
#define RTE_PMD_RING_MAX_RX_RINGS 16
#define RTE_PMD_RING_MAX_TX_RINGS 16
/* QEDE PMD defines */
#define RTE_LIBRTE_QEDE_FW ""
/* DLB2 defines */
// RTE_LIBRTE_PMD_DLB2_QUELL_STATS is not set
#endif /* _RTE_CONFIG_H_ */

75
Programs/hugePage/common/rte_debug.h Normal file
View File

@@ -0,0 +1,75 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_DEBUG_H_
#define _RTE_DEBUG_H_
/**
* @file
*
* Debug Functions in RTE
*
* This file defines a generic API for debug operations. Part of
* the implementation is architecture-specific.
*/
#include "rte_log.h"
#include "rte_branch_prediction.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* Dump the stack of the calling core to the standard error.
*/
void rte_dump_stack(void);
/**
* Provide notification of a critical non-recoverable error and terminate
* execution abnormally.
*
* Display the format string and its expanded arguments (printf-like).
*
* In a linux environment, this function dumps the stack and calls
* abort() resulting in a core dump if enabled.
*
* The function never returns.
*
* @param ...
* The format string, followed by the variable list of arguments.
*/
#define rte_panic(...) rte_panic_(__func__, __VA_ARGS__, "dummy")
#define rte_panic_(func, format, ...) __rte_panic(func, format "%.0s", __VA_ARGS__)
#ifdef RTE_ENABLE_ASSERT
#define RTE_ASSERT(exp) RTE_VERIFY(exp)
#else
#define RTE_ASSERT(exp) do {} while (0)
#endif
#define RTE_VERIFY(exp) do { \
if (unlikely(!(exp))) \
rte_panic("line %d\tassert \"%s\" failed\n", __LINE__, #exp); \
} while (0)
/*
* Provide notification of a critical non-recoverable error and stop.
*
* This function should not be called directly. Refer to rte_panic() macro
* documentation.
*/
void __rte_panic(const char *funcname , const char *format, ...)
#ifdef __GNUC__
#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 2))
__rte_cold
#endif
#endif
__rte_noreturn
__rte_format_printf(2, 3);
#ifdef __cplusplus
}
#endif
#endif /* _RTE_DEBUG_H_ */

563
Programs/hugePage/common/rte_dev.h Normal file
View File

@@ -0,0 +1,563 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2014 6WIND S.A.
*/
#ifndef _RTE_DEV_H_
#define _RTE_DEV_H_
/**
* @file
*
* RTE PMD Registration Interface
*
* This file manages the list of device drivers.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include "rte_config.h"
#include "rte_common.h"
#include "rte_compat.h"
#include "rte_log.h"
struct rte_bus;
struct rte_devargs;
struct rte_device;
struct rte_driver;
/**
* The device event type.
*/
enum rte_dev_event_type {
RTE_DEV_EVENT_ADD, /**< device being added */
RTE_DEV_EVENT_REMOVE, /**< device being removed */
RTE_DEV_EVENT_MAX /**< max value of this enum */
};
typedef void (*rte_dev_event_cb_fn)(const char *device_name,
enum rte_dev_event_type event,
void *cb_arg);
/* Macros to check for invalid function pointers */
#define RTE_FUNC_PTR_OR_ERR_RET(func, retval) RTE_DEPRECATED(RTE_FUNC_PTR_OR_ERR_RET) \
do { \
if ((func) == NULL) \
return retval; \
} while (0)
#define RTE_FUNC_PTR_OR_RET(func) RTE_DEPRECATED(RTE_FUNC_PTR_OR_RET) \
do { \
if ((func) == NULL) \
return; \
} while (0)
/**
* Device policies.
*/
enum rte_dev_policy {
RTE_DEV_ALLOWED,
RTE_DEV_BLOCKED,
};
/**
* A generic memory resource representation.
*/
struct rte_mem_resource {
uint64_t phys_addr; /**< Physical address, 0 if not resource. */
uint64_t len; /**< Length of the resource. */
void *addr; /**< Virtual address, NULL when not mapped. */
};
/**
* Retrieve a driver name.
*
* @param driver
* A pointer to a driver structure.
* @return
* A pointer to the driver name string.
*/
const char *
rte_driver_name(const struct rte_driver *driver);
/**
* Retrieve a device bus.
*
* @param dev
* A pointer to a device structure.
* @return
* A pointer to this device bus.
*/
const struct rte_bus *
rte_dev_bus(const struct rte_device *dev);
/**
* Retrieve bus specific information for a device.
*
* @param dev
* A pointer to a device structure.
* @return
* A string describing this device or NULL if none is available.
*/
const char *
rte_dev_bus_info(const struct rte_device *dev);
/**
* Retrieve a device arguments.
*
* @param dev
* A pointer to a device structure.
* @return
* A pointer to this device devargs.
*/
const struct rte_devargs *
rte_dev_devargs(const struct rte_device *dev);
/**
* Retrieve a device driver.
*
* @param dev
* A pointer to a device structure.
* @return
* A pointer to this device driver.
*/
const struct rte_driver *
rte_dev_driver(const struct rte_device *dev);
/**
* Retrieve a device name.
*
* @param dev
* A pointer to a device structure.
* @return
* A pointer to this device name.
*/
const char *
rte_dev_name(const struct rte_device *dev);
/**
* Retrieve a device numa node.
*
* @param dev
* A pointer to a device structure.
* @return
* A pointer to this device numa node.
*/
int
rte_dev_numa_node(const struct rte_device *dev);
/*
* Internal identifier length
* Sufficiently large to allow for UUID or PCI address
*/
#define RTE_DEV_NAME_MAX_LEN 64
/**
* Query status of a device.
*
* @param dev
* Generic device pointer.
* @return
* (int)true if already probed successfully, 0 otherwise.
*/
int rte_dev_is_probed(const struct rte_device *dev);
/**
* Hotplug add a given device to a specific bus.
*
* In multi-process, it will request other processes to add the same device.
* A failure, in any process, will rollback the action
*
* @param busname
* The bus name the device is added to.
* @param devname
* The device name. Based on this device name, eal will identify a driver
* capable of handling it and pass it to the driver probing function.
* @param drvargs
* Device arguments to be passed to the driver.
* @return
* 0 on success, negative on error.
*/
int rte_eal_hotplug_add(const char *busname, const char *devname,
const char *drvargs);
/**
* Add matching devices.
*
* In multi-process, it will request other processes to add the same device.
* A failure, in any process, will rollback the action
*
* @param devargs
* Device arguments including bus, class and driver properties.
* @return
* 0 on success, negative on error.
*/
int rte_dev_probe(const char *devargs);
/**
* Hotplug remove a given device from a specific bus.
*
* In multi-process, it will request other processes to remove the same device.
* A failure, in any process, will rollback the action
*
* @param busname
* The bus name the device is removed from.
* @param devname
* The device name being removed.
* @return
* 0 on success, negative on error.
*/
int rte_eal_hotplug_remove(const char *busname, const char *devname);
/**
* Remove one device.
*
* In multi-process, it will request other processes to remove the same device.
* A failure, in any process, will rollback the action
*
* @param dev
* Data structure of the device to remove.
* @return
* 0 on success, negative on error.
*/
int rte_dev_remove(struct rte_device *dev);
/**
* Device comparison function.
*
* This type of function is used to compare an rte_device with arbitrary
* data.
*
* @param dev
* Device handle.
*
* @param data
* Data to compare against. The type of this parameter is determined by
* the kind of comparison performed by the function.
*
* @return
* 0 if the device matches the data.
* !0 if the device does not match.
* <0 if ordering is possible and the device is lower than the data.
* >0 if ordering is possible and the device is greater than the data.
*/
typedef int (*rte_dev_cmp_t)(const struct rte_device *dev, const void *data);
#define RTE_PMD_EXPORT_NAME_ARRAY(n, idx) n##idx[]
#define RTE_PMD_EXPORT_NAME(name, idx) \
static const char RTE_PMD_EXPORT_NAME_ARRAY(this_pmd_name, idx) \
__rte_used = RTE_STR(name)
#define DRV_EXP_TAG(name, tag) __##name##_##tag
#define RTE_PMD_REGISTER_PCI_TABLE(name, table) \
static const char DRV_EXP_TAG(name, pci_tbl_export)[] __rte_used = \
RTE_STR(table)
#define RTE_PMD_REGISTER_PARAM_STRING(name, str) \
static const char DRV_EXP_TAG(name, param_string_export)[] \
__rte_used = str
/**
* Advertise the list of kernel modules required to run this driver
*
* This string lists the kernel modules required for the devices
* associated to a PMD. The format of each line of the string is:
* "<device-pattern> <kmod-expression>".
*
* The possible formats for the device pattern are:
* "*" all devices supported by this driver
* "pci:*" all PCI devices supported by this driver
* "pci:v8086:d*:sv*:sd*" all PCI devices supported by this driver
* whose vendor id is 0x8086.
*
* The format of the kernel modules list is a parenthesized expression
* containing logical-and (&) and logical-or (|).
*
* The device pattern and the kmod expression are separated by a space.
*
* Example:
* - "* igb_uio | uio_pci_generic | vfio"
*/
#define RTE_PMD_REGISTER_KMOD_DEP(name, str) \
static const char DRV_EXP_TAG(name, kmod_dep_export)[] \
__rte_used = str
/**
* Iteration context.
*
* This context carries over the current iteration state.
*/
struct rte_dev_iterator {
const char *dev_str; /**< device string. */
const char *bus_str; /**< bus-related part of device string. */
const char *cls_str; /**< class-related part of device string. */
struct rte_bus *bus; /**< bus handle. */
struct rte_class *cls; /**< class handle. */
struct rte_device *device; /**< current position. */
void *class_device; /**< additional specialized context. */
};
/**
* Device iteration function.
*
* Find the next device matching properties passed in parameters.
* The function takes an additional ``start`` parameter, that is
* used as starting context when relevant.
*
* The function returns the current element in the iteration.
* This return value will potentially be used as a start parameter
* in subsequent calls to the function.
*
* The additional iterator parameter is only there if a specific
* implementation needs additional context. It must not be modified by
* the iteration function itself.
*
* @param start
* Starting iteration context.
*
* @param devstr
* Device description string.
*
* @param it
* Device iterator.
*
* @return
* The address of the current element matching the device description
* string.
*/
typedef void *(*rte_dev_iterate_t)(const void *start,
const char *devstr,
const struct rte_dev_iterator *it);
/**
* Initializes a device iterator.
*
* This iterator allows accessing a list of devices matching a criteria.
* The device matching is made among all buses and classes currently registered,
* filtered by the device description given as parameter.
*
* This function will not allocate any memory. It is safe to stop the
* iteration at any moment and let the iterator go out of context.
*
* @param it
* Device iterator handle.
*
* @param str
* Device description string.
*
* @return
* 0 on successful initialization.
* <0 on error.
*/
__rte_experimental
int
rte_dev_iterator_init(struct rte_dev_iterator *it, const char *str);
/**
* Iterates on a device iterator.
*
* Generates a new rte_device handle corresponding to the next element
* in the list described in comprehension by the iterator.
*
* The next object is returned, and the iterator is updated.
*
* @param it
* Device iterator handle.
*
* @return
* An rte_device handle if found.
* NULL if an error occurred (rte_errno is set).
* NULL if no device could be found (rte_errno is not set).
*/
__rte_experimental
struct rte_device *
rte_dev_iterator_next(struct rte_dev_iterator *it);
#define RTE_DEV_FOREACH(dev, devstr, it) \
for (rte_dev_iterator_init(it, devstr), \
dev = rte_dev_iterator_next(it); \
dev != NULL; \
dev = rte_dev_iterator_next(it))
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* It registers the callback for the specific device.
* Multiple callbacks can be registered at the same time.
*
* @param device_name
* The device name, that is the param name of the struct rte_device,
* null value means for all devices.
* @param cb_fn
* callback address.
* @param cb_arg
* address of parameter for callback.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
__rte_experimental
int
rte_dev_event_callback_register(const char *device_name,
rte_dev_event_cb_fn cb_fn,
void *cb_arg);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* It unregisters the callback according to the specified device.
*
* @param device_name
* The device name, that is the param name of the struct rte_device,
* null value means for all devices and their callbacks.
* @param cb_fn
* callback address.
* @param cb_arg
* address of parameter for callback, (void *)-1 means to remove all
* registered which has the same callback address.
*
* @return
* - On success, return the number of callback entities removed.
* - On failure, a negative value.
*/
__rte_experimental
int
rte_dev_event_callback_unregister(const char *device_name,
rte_dev_event_cb_fn cb_fn,
void *cb_arg);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Executes all the user application registered callbacks for
* the specific device.
*
* @param device_name
* The device name.
* @param event
* the device event type.
*/
__rte_experimental
void
rte_dev_event_callback_process(const char *device_name,
enum rte_dev_event_type event);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Start the device event monitoring.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
__rte_experimental
int
rte_dev_event_monitor_start(void);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Stop the device event monitoring.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
__rte_experimental
int
rte_dev_event_monitor_stop(void);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Enable hotplug handling for devices.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
__rte_experimental
int
rte_dev_hotplug_handle_enable(void);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Disable hotplug handling for devices.
*
* @return
* - On success, zero.
* - On failure, a negative value.
*/
__rte_experimental
int
rte_dev_hotplug_handle_disable(void);
/**
* Device level DMA map function.
* After a successful call, the memory segment will be mapped to the
* given device.
*
* @note: Memory must be registered in advance using rte_extmem_* APIs.
*
* @param dev
* Device pointer.
* @param addr
* Virtual address to map.
* @param iova
* IOVA address to map.
* @param len
* Length of the memory segment being mapped.
*
* @return
* 0 if mapping was successful.
* Negative value and rte_errno is set otherwise.
*/
__rte_experimental
int
rte_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len);
/**
* Device level DMA unmap function.
* After a successful call, the memory segment will no longer be
* accessible by the given device.
*
* @note: Memory must be registered in advance using rte_extmem_* APIs.
*
* @param dev
* Device pointer.
* @param addr
* Virtual address to unmap.
* @param iova
* IOVA address to unmap.
* @param len
* Length of the memory segment being mapped.
*
* @return
* 0 if un-mapping was successful.
* Negative value and rte_errno is set otherwise.
*/
__rte_experimental
int
rte_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
size_t len);
#ifdef __cplusplus
}
#endif
#endif /* _RTE_DEV_H_ */

272
Programs/hugePage/common/rte_devargs.h Normal file
View File

@@ -0,0 +1,272 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2014 6WIND S.A.
*/
#ifndef _RTE_DEVARGS_H_
#define _RTE_DEVARGS_H_
/**
* @file
*
* RTE devargs: list of devices and their user arguments
*
* This file stores a list of devices and their arguments given by
* the user when a DPDK application is started. These devices can be PCI
* devices or virtual devices. These devices are stored at startup in a
* list of rte_devargs structures.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include "rte_compat.h"
#include "rte_dev.h"
struct rte_bus;
/**
* Bus type key in global devargs syntax.
*
* Legacy devargs parser doesn't use this key as bus type
* is resolved as first optional value separated by ":".
*/
#define RTE_DEVARGS_KEY_BUS "bus"
/**
* Class type key in global devargs syntax.
*
* Legacy devargs parser doesn't parse class type. PMD is
* encouraged to use this key to resolve class type.
*/
#define RTE_DEVARGS_KEY_CLASS "class"
/**
* Driver type key in global devargs syntax.
*
* Legacy devargs parser doesn't parse driver type. PMD is
* encouraged to use this key to resolve driver type.
*/
#define RTE_DEVARGS_KEY_DRIVER "driver"
/**
* Type of generic device
*/
enum rte_devtype {
RTE_DEVTYPE_ALLOWED,
RTE_DEVTYPE_BLOCKED,
RTE_DEVTYPE_VIRTUAL,
};
/**
* Structure that stores a device given by the user with its arguments
*
* A user device is a physical or a virtual device given by the user to
* the DPDK application at startup through command line arguments.
*
* The structure stores the configuration of the device, its PCI
* identifier if it's a PCI device or the driver name if it's a virtual
* device.
*/
struct rte_devargs {
/** Next in list. */
RTE_TAILQ_ENTRY(rte_devargs) next;
/** Type of device. */
enum rte_devtype type;
/** Device policy. */
enum rte_dev_policy policy;
/** Name of the device. */
char name[RTE_DEV_NAME_MAX_LEN];
RTE_STD_C11
union {
const char *args; /**< legacy name. */
const char *drv_str; /**< driver-related part of device string. */
};
struct rte_bus *bus; /**< bus handle. */
struct rte_class *cls; /**< class handle. */
const char *bus_str; /**< bus-related part of device string. */
const char *cls_str; /**< class-related part of device string. */
char *data; /**< raw string including bus, class and driver parts. */
};
/**
* Parse a device string.
*
* Verify that a bus is capable of handling the device passed
* in argument. Store which bus will handle the device, its name
* and the eventual device parameters.
*
* The syntax is:
*
* bus:device_identifier,arg1=val1,arg2=val2
*
* where "bus:" is the bus name followed by any character separator.
* The bus name is optional. If no bus name is specified, each bus
* will attempt to recognize the device identifier. The first one
* to succeed will be used.
*
* Examples:
*
* pci:0000:05.00.0,arg=val
* 05.00.0,arg=val
* vdev:net_ring0
*
* @param da
* The devargs structure holding the device information.
*
* @param dev
* String describing a device.
*
* @return
* - 0 on success.
* - Negative errno on error.
*/
int
rte_devargs_parse(struct rte_devargs *da, const char *dev);
/**
* Parse a device string.
*
* Verify that a bus is capable of handling the device passed
* in argument. Store which bus will handle the device, its name
* and the eventual device parameters.
*
* The device string is built with a printf-like syntax.
*
* The syntax is:
*
* bus:device_identifier,arg1=val1,arg2=val2
*
* where "bus:" is the bus name followed by any character separator.
* The bus name is optional. If no bus name is specified, each bus
* will attempt to recognize the device identifier. The first one
* to succeed will be used.
*
* Examples:
*
* pci:0000:05.00.0,arg=val
* 05.00.0,arg=val
* vdev:net_ring0
*
* @param da
* The devargs structure holding the device information.
* @param format
* Format string describing a device.
*
* @return
* - 0 on success.
* - Negative errno on error.
*/
int
rte_devargs_parsef(struct rte_devargs *da,
const char *format, ...)
__rte_format_printf(2, 0);
/**
* Free resources in devargs.
*
* @param da
* The devargs structure holding the device information.
*/
__rte_experimental
void
rte_devargs_reset(struct rte_devargs *da);
/**
* Insert an rte_devargs in the global list.
*
* @param da
* The devargs structure to insert.
* If a devargs for the same device is already inserted,
* it will be updated and returned. It means *da pointer can change.
*
* @return
* - 0 on success
* - Negative on error.
*/
int
rte_devargs_insert(struct rte_devargs **da);
/**
* Add a device to the user device list
* See rte_devargs_parse() for details.
*
* @param devtype
* The type of the device.
* @param devargs_str
* The arguments as given by the user.
*
* @return
* - 0 on success
* - A negative value on error
*/
int rte_devargs_add(enum rte_devtype devtype, const char *devargs_str);
/**
* Remove a device from the user device list.
* Its resources are freed.
* If the devargs cannot be found, nothing happens.
*
* @param devargs
* The instance or a copy of devargs to remove.
*
* @return
* 0 on success.
* <0 on error.
* >0 if the devargs was not within the user device list.
*/
int rte_devargs_remove(struct rte_devargs *devargs);
/**
* Count the number of user devices of a specified type
*
* @param devtype
* The type of the devices to counted.
*
* @return
* The number of devices.
*/
unsigned int
rte_devargs_type_count(enum rte_devtype devtype);
/**
* This function dumps the list of user device and their arguments.
*
* @param f
* A pointer to a file for output
*/
void rte_devargs_dump(FILE *f);
/**
* Find next rte_devargs matching the provided bus name.
*
* @param busname
* Limit the iteration to devargs related to buses
* matching this name.
* Will return any next rte_devargs if NULL.
*
* @param start
* Starting iteration point. The iteration will start at
* the first rte_devargs if NULL.
*
* @return
* Next rte_devargs entry matching the requested bus,
* NULL if there is none.
*/
struct rte_devargs *
rte_devargs_next(const char *busname, const struct rte_devargs *start);
/**
* Iterate over all rte_devargs for a specific bus.
*/
#define RTE_EAL_DEVARGS_FOREACH(busname, da) \
for (da = rte_devargs_next(busname, NULL); \
da != NULL; \
da = rte_devargs_next(busname, da)) \
#ifdef __cplusplus
}
#endif
#endif /* _RTE_DEVARGS_H_ */

524
Programs/hugePage/common/rte_eal.h Normal file
View File

@@ -0,0 +1,524 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2018 Intel Corporation
*/
#ifndef _RTE_EAL_H_
#define _RTE_EAL_H_
/**
* @file
*
* EAL Configuration API
*/
#include <stdint.h>
#include <time.h>
#include "rte_config.h"
#include "rte_compat.h"
#include "rte_per_lcore.h"
#include "rte_uuid.h"
#include "rte_pci_dev_feature_defs.h"
#ifdef __cplusplus
extern "C" {
#endif
#define RTE_MAGIC 19820526 /**< Magic number written by the main partition when ready. */
/* Maximum thread_name length. */
#define RTE_MAX_THREAD_NAME_LEN 16
/**
* The type of process in a linux, multi-process setup
*/
enum rte_proc_type_t {
RTE_PROC_AUTO = -1, /* allow auto-detection of primary/secondary */
RTE_PROC_PRIMARY = 0, /* set to zero, so primary is the default */
RTE_PROC_SECONDARY,
RTE_PROC_INVALID
};
/**
* Get the process type in a multi-process setup
*
* @return
* The process type
*/
enum rte_proc_type_t rte_eal_process_type(void);
/**
* Request iopl privilege for all RPL.
*
* This function should be called by pmds which need access to ioports.
* @return
* - On success, returns 0.
* - On failure, returns -1.
*/
int rte_eal_iopl_init(void);
/**
* Initialize the Environment Abstraction Layer (EAL).
*
* This function is to be executed on the MAIN lcore only, as soon
* as possible in the application's main() function.
* It puts the WORKER lcores in the WAIT state.
*
* @param argc
* A non-negative value. If it is greater than 0, the array members
* for argv[0] through argv[argc] (non-inclusive) shall contain pointers
* to strings.
* @param argv
* An array of strings. The contents of the array, as well as the strings
* which are pointed to by the array, may be modified by this function.
* @return
* - On success, the number of parsed arguments, which is greater or
* equal to zero. After the call to rte_eal_init(),
* all arguments argv[x] with x < ret may have been modified by this
* function call and should not be further interpreted by the
* application. The EAL does not take any ownership of the memory used
* for either the argv array, or its members.
* - On failure, -1 and rte_errno is set to a value indicating the cause
* for failure. In some instances, the application will need to be
* restarted as part of clearing the issue.
*
* Error codes returned via rte_errno:
* EACCES indicates a permissions issue.
*
* EAGAIN indicates either a bus or system resource was not available,
* setup may be attempted again.
*
* EALREADY indicates that the rte_eal_init function has already been
* called, and cannot be called again.
*
* EFAULT indicates the tailq configuration name was not found in
* memory configuration.
*
* EINVAL indicates invalid parameters were passed as argv/argc.
*
* ENOMEM indicates failure likely caused by an out-of-memory condition.
*
* ENODEV indicates memory setup issues.
*
* ENOTSUP indicates that the EAL cannot initialize on this system.
*
* EPROTO indicates that the PCI bus is either not present, or is not
* readable by the eal.
*
* ENOEXEC indicates that a service core failed to launch successfully.
*/
int rte_eal_init(int argc, char **argv);
/**
* Clean up the Environment Abstraction Layer (EAL)
*
* This function must be called to release any internal resources that EAL has
* allocated during rte_eal_init(). After this call, no DPDK function calls may
* be made. It is expected that common usage of this function is to call it
* just before terminating the process.
*
* @return
* - 0 Successfully released all internal EAL resources.
* - -EFAULT There was an error in releasing all resources.
*/
int rte_eal_cleanup(void);
/**
* Check if a primary process is currently alive
*
* This function returns true when a primary process is currently
* active.
*
* @param config_file_path
* The config_file_path argument provided should point at the location
* that the primary process will create its config file. If NULL, the default
* config file path is used.
*
* @return
* - If alive, returns 1.
* - If dead, returns 0.
*/
int rte_eal_primary_proc_alive(const char *config_file_path);
/**
* Disable multiprocess.
*
* This function can be called to indicate that multiprocess won't be used for
* the rest of the application life.
*
* @return
* - true if called from a primary process that had no secondary processes
* attached,
* - false, otherwise.
*/
bool rte_mp_disable(void);
#define RTE_MP_MAX_FD_NUM 8 /* The max amount of fds */
#define RTE_MP_MAX_NAME_LEN 64 /* The max length of action name */
#define RTE_MP_MAX_PARAM_LEN 256 /* The max length of param */
struct rte_mp_msg {
char name[RTE_MP_MAX_NAME_LEN];
int len_param;
int num_fds;
uint8_t param[RTE_MP_MAX_PARAM_LEN];
int fds[RTE_MP_MAX_FD_NUM];
};
struct rte_mp_reply {
int nb_sent;
int nb_received;
struct rte_mp_msg *msgs; /* caller to free */
};
/**
* Action function typedef used by other components.
*
* As we create socket channel for primary/secondary communication, use
* this function typedef to register action for coming messages.
*
* @note When handling IPC request callbacks, the reply must be sent even in
* cases of error handling. Simply returning success or failure will *not*
* send a response to the requestor.
* Implementation of error signalling mechanism is up to the application.
*
* @note No memory allocations should take place inside the callback.
*/
typedef int (*rte_mp_t)(const struct rte_mp_msg *msg, const void *peer);
/**
* Asynchronous reply function typedef used by other components.
*
* As we create socket channel for primary/secondary communication, use
* this function typedef to register action for coming responses to asynchronous
* requests.
*
* @note When handling IPC request callbacks, the reply must be sent even in
* cases of error handling. Simply returning success or failure will *not*
* send a response to the requestor.
* Implementation of error signalling mechanism is up to the application.
*
* @note No memory allocations should take place inside the callback.
*/
typedef int (*rte_mp_async_reply_t)(const struct rte_mp_msg *request,
const struct rte_mp_reply *reply);
/**
* Register an action function for primary/secondary communication.
*
* Call this function to register an action, if the calling component wants
* to response the messages from the corresponding component in its primary
* process or secondary processes.
*
* @note IPC may be unsupported in certain circumstances, so caller should check
* for ENOTSUP error.
*
* @param name
* The name argument plays as the nonredundant key to find the action.
*
* @param action
* The action argument is the function pointer to the action function.
*
* @return
* - 0 on success.
* - (<0) on failure.
*/
int
rte_mp_action_register(const char *name, rte_mp_t action);
/**
* Unregister an action function for primary/secondary communication.
*
* Call this function to unregister an action if the calling component does
* not want to response the messages from the corresponding component in its
* primary process or secondary processes.
*
* @note IPC may be unsupported in certain circumstances, so caller should check
* for ENOTSUP error.
*
* @param name
* The name argument plays as the nonredundant key to find the action.
*
*/
void
rte_mp_action_unregister(const char *name);
/**
* Send a message to the peer process.
*
* This function will send a message which will be responded by the action
* identified by name in the peer process.
*
* @param msg
* The msg argument contains the customized message.
*
* @return
* - On success, return 0.
* - On failure, return -1, and the reason will be stored in rte_errno.
*/
int
rte_mp_sendmsg(struct rte_mp_msg *msg);
/**
* Send a request to the peer process and expect a reply.
*
* This function sends a request message to the peer process, and will
* block until receiving reply message from the peer process.
*
* @note The caller is responsible to free reply->replies.
*
* @note This API must not be used inside memory-related or IPC callbacks, and
* no memory allocations should take place inside such callback.
*
* @note IPC may be unsupported in certain circumstances, so caller should check
* for ENOTSUP error.
*
* @param req
* The req argument contains the customized request message.
*
* @param reply
* The reply argument will be for storing all the replied messages;
* the caller is responsible for free reply->msgs.
*
* @param ts
* The ts argument specifies how long we can wait for the peer(s) to reply.
*
* @return
* - On success, return 0.
* - On failure, return -1, and the reason will be stored in rte_errno.
*/
int
rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply,
const struct timespec *ts);
/**
* Send a request to the peer process and expect a reply in a separate callback.
*
* This function sends a request message to the peer process, and will not
* block. Instead, reply will be received in a separate callback.
*
* @note IPC may be unsupported in certain circumstances, so caller should check
* for ENOTSUP error.
*
* @param req
* The req argument contains the customized request message.
*
* @param ts
* The ts argument specifies how long we can wait for the peer(s) to reply.
*
* @param clb
* The callback to trigger when all responses for this request have arrived.
*
* @return
* - On success, return 0.
* - On failure, return -1, and the reason will be stored in rte_errno.
*/
int
rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts,
rte_mp_async_reply_t clb);
/**
* Send a reply to the peer process.
*
* This function will send a reply message in response to a request message
* received previously.
*
* @note When handling IPC request callbacks, the reply must be sent even in
* cases of error handling. Simply returning success or failure will *not*
* send a response to the requestor.
* Implementation of error signalling mechanism is up to the application.
*
* @param msg
* The msg argument contains the customized message.
*
* @param peer
* The peer argument is the pointer to the peer socket path.
*
* @return
* - On success, return 0.
* - On failure, return -1, and the reason will be stored in rte_errno.
*/
int
rte_mp_reply(struct rte_mp_msg *msg, const char *peer);
/**
* Usage function typedef used by the application usage function.
*
* Use this function typedef to define and call rte_set_application_usage_hook()
* routine.
*/
typedef void (*rte_usage_hook_t)(const char * prgname);
/**
* Add application usage routine callout from the eal_usage() routine.
*
* This function allows the application to include its usage message
* in the EAL system usage message. The routine rte_set_application_usage_hook()
* needs to be called before the rte_eal_init() routine in the application.
*
* This routine is optional for the application and will behave as if the set
* routine was never called as the default behavior.
*
* @param usage_func
* The func argument is a function pointer to the application usage routine.
* Called function is defined using rte_usage_hook_t typedef, which is of
* the form void rte_usage_func(const char * prgname).
*
* Calling this routine with a NULL value will reset the usage hook routine and
* return the current value, which could be NULL.
* @return
* - Returns the current value of the rte_application_usage pointer to allow
* the caller to daisy chain the usage routines if needing more then one.
*/
rte_usage_hook_t
rte_set_application_usage_hook(rte_usage_hook_t usage_func);
/**
* Whether EAL is using huge pages (disabled by --no-huge option).
* The no-huge mode is not compatible with all drivers or features.
*
* @return
* Nonzero if hugepages are enabled.
*/
int rte_eal_has_hugepages(void);
/**
* Whether EAL is using PCI bus.
* Disabled by --no-pci option.
*
* @return
* Nonzero if the PCI bus is enabled.
*/
int rte_eal_has_pci(void);
/**
* Whether the EAL was asked to create UIO device.
*
* @return
* Nonzero if true.
*/
int rte_eal_create_uio_dev(void);
/**
* The user-configured vfio interrupt mode.
*
* @return
* Interrupt mode configured with the command line,
* RTE_INTR_MODE_NONE by default.
*/
enum rte_intr_mode rte_eal_vfio_intr_mode(void);
/**
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
*
* Copy the user-configured vfio VF token.
*
* @param vf_token
* vfio VF token configured with the command line is copied
* into this parameter, zero uuid by default.
*/
__rte_experimental
void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
/**
* A wrap API for syscall gettid.
*
* @return
* On success, returns the thread ID of calling process.
* It is always successful.
*/
int rte_sys_gettid(void);
RTE_DECLARE_PER_LCORE(int, _thread_id);
/**
* Get system unique thread id.
*
* @return
* On success, returns the thread ID of calling process.
* It is always successful.
*/
static inline int rte_gettid(void)
{
if (RTE_PER_LCORE(_thread_id) == -1)
RTE_PER_LCORE(_thread_id) = rte_sys_gettid();
return RTE_PER_LCORE(_thread_id);
}
/**
* Get the OS-specific EAL base address.
*
* @return
* The base address.
*/
__rte_internal
uint64_t rte_eal_get_baseaddr(void);
/**
* IOVA mapping mode.
*
* IOVA mapping mode is iommu programming mode of a device.
* That device (for example: IOMMU backed DMA device) based
* on rte_iova_mode will generate physical or virtual address.
*
*/
enum rte_iova_mode {
RTE_IOVA_DC = 0, /* Don't care mode */
RTE_IOVA_PA = (1 << 0), /* DMA using physical address */
RTE_IOVA_VA = (1 << 1) /* DMA using virtual address */
};
/**
* Get the iova mode
*
* @return
* enum rte_iova_mode value.
*/
enum rte_iova_mode rte_eal_iova_mode(void);
/**
* Get user provided pool ops name for mbuf
*
* @return
* returns user provided pool ops name.
*/
const char *
rte_eal_mbuf_user_pool_ops(void);
/**
* Get the runtime directory of DPDK
*
* @return
* The runtime directory path of DPDK
*/
const char *
rte_eal_get_runtime_dir(void);
/**
* Convert a string describing a mask of core ids into an array of core ids.
*
* On success, the passed array is filled with the orders of the core ids
* present in the mask (-1 indicating that a core id is absent).
* For example, passing a 0xa coremask results in cores[1] = 0, cores[3] = 1,
* and the rest of the array is set to -1.
*
* @param coremask
* A string describing a mask of core ids.
* @param cores
* An array where to store the core ids orders.
* This array must be at least RTE_MAX_LCORE large.
* @return
* 0 on success, -1 if the string content was invalid.
*/
__rte_internal
int
rte_eal_parse_coremask(const char *coremask, int *cores);
#ifdef __cplusplus
}
#endif
#endif /* _RTE_EAL_H_ */

116
Programs/hugePage/common/rte_eal_memconfig.h Normal file
View File

@@ -0,0 +1,116 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#ifndef _RTE_EAL_MEMCONFIG_H_
#define _RTE_EAL_MEMCONFIG_H_
#include <stdbool.h>
/**
* @file
*
* This API allows access to EAL shared memory configuration through an API.
*/
#ifdef __cplusplus
extern "C" {
#endif
/**
* Lock the internal EAL shared memory configuration for shared access.
*/
void
rte_mcfg_mem_read_lock(void);
/**
* Unlock the internal EAL shared memory configuration for shared access.
*/
void
rte_mcfg_mem_read_unlock(void);
/**
* Lock the internal EAL shared memory configuration for exclusive access.
*/
void
rte_mcfg_mem_write_lock(void);
/**
* Unlock the internal EAL shared memory configuration for exclusive access.
*/
void
rte_mcfg_mem_write_unlock(void);
/**
* Lock the internal EAL TAILQ list for shared access.
*/
void
rte_mcfg_tailq_read_lock(void);
/**
* Unlock the internal EAL TAILQ list for shared access.
*/
void
rte_mcfg_tailq_read_unlock(void);
/**
* Lock the internal EAL TAILQ list for exclusive access.
*/
void
rte_mcfg_tailq_write_lock(void);
/**
* Unlock the internal EAL TAILQ list for exclusive access.
*/
void
rte_mcfg_tailq_write_unlock(void);
/**
* Lock the internal EAL Mempool list for shared access.
*/
void
rte_mcfg_mempool_read_lock(void);
/**
* Unlock the internal EAL Mempool list for shared access.
*/
void
rte_mcfg_mempool_read_unlock(void);
/**
* Lock the internal EAL Mempool list for exclusive access.
*/
void
rte_mcfg_mempool_write_lock(void);
/**
* Unlock the internal EAL Mempool list for exclusive access.
*/
void
rte_mcfg_mempool_write_unlock(void);
/**
* Lock the internal EAL Timer Library lock for exclusive access.
*/
void
rte_mcfg_timer_lock(void);
/**
* Unlock the internal EAL Timer Library lock for exclusive access.
*/
void
rte_mcfg_timer_unlock(void);
/**
* If true, pages are put in single files (per memseg list),
* as opposed to creating a file per page.
*/
bool
rte_mcfg_get_single_file_segments(void);
#ifdef __cplusplus
}
#endif
#endif /*__RTE_EAL_MEMCONFIG_H_*/

98
Programs/hugePage/common/rte_eal_paging.h Normal file
View File

@@ -0,0 +1,98 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2020 Dmitry Kozlyuk
*/
#include <stdint.h>
#include "rte_compat.h"
/**
* @file
* @internal
*
* Wrappers for OS facilities related to memory paging, used across DPDK.
*/
/** Memory protection flags. */
enum rte_mem_prot {
RTE_PROT_READ = 1 << 0, /**< Read access. */
RTE_PROT_WRITE = 1 << 1, /**< Write access. */
RTE_PROT_EXECUTE = 1 << 2 /**< Code execution. */
};
/** Additional flags for memory mapping. */
enum rte_map_flags {
/** Changes to the mapped memory are visible to other processes. */
RTE_MAP_SHARED = 1 << 0,
/** Mapping is not backed by a regular file. */
RTE_MAP_ANONYMOUS = 1 << 1,
/** Copy-on-write mapping, changes are invisible to other processes. */
RTE_MAP_PRIVATE = 1 << 2,
/**
* Force mapping to the requested address. This flag should be used
* with caution, because to fulfill the request implementation
* may remove all other mappings in the requested region. However,
* it is not required to do so, thus mapping with this flag may fail.
*/
RTE_MAP_FORCE_ADDRESS = 1 << 3
};
/**
* Map a portion of an opened file or the page file into memory.
*
* This function is similar to POSIX mmap(3) with common MAP_ANONYMOUS
* extension, except for the return value.
*
* @param requested_addr
* Desired virtual address for mapping. Can be NULL to let OS choose.
* @param size
* Size of the mapping in bytes.
* @param prot
* Protection flags, a combination of rte_mem_prot values.
* @param flags
* Additional mapping flags, a combination of rte_map_flags.
* @param fd
* Mapped file descriptor. Can be negative for anonymous mapping.
* @param offset
* Offset of the mapped region in fd. Must be 0 for anonymous mappings.
* @return
* Mapped address or NULL on failure and rte_errno is set to OS error.
*/
__rte_internal
void *
rte_mem_map(void *requested_addr, size_t size, int prot, int flags,
int fd, uint64_t offset);
/**
* OS-independent implementation of POSIX munmap(3).
*/
__rte_internal
int
rte_mem_unmap(void *virt, size_t size);
/**
* Get system page size. This function never fails.
*
* @return
* Page size in bytes.
*/
__rte_internal
size_t
rte_mem_page_size(void);
/**
* Lock in physical memory all pages crossed by the address region.
*
* @param virt
* Base virtual address of the region.
* @param size
* Size of the region.
* @return
* 0 on success, negative on error.
*
* @see rte_mem_page_size() to retrieve the page size.
* @see rte_mem_lock_page() to lock an entire single page.
*/
__rte_internal
int
rte_mem_lock(const void *virt, size_t size);

286
Programs/hugePage/common/rte_eal_trace.h Normal file
View File

@@ -0,0 +1,286 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2020 Marvell International Ltd.
*/
#ifndef _RTE_EAL_TRACE_H_
#define _RTE_EAL_TRACE_H_
/**
* @file
*
* API for EAL trace support
*/
#ifdef __cplusplus
extern "C" {
#endif
#include "rte_alarm.h"
#include "rte_interrupts.h"
#include "rte_trace_point.h"
#include "eal_interrupts.h"
/* Alarm */
RTE_TRACE_POINT(
rte_eal_trace_alarm_set,
RTE_TRACE_POINT_ARGS(uint64_t us, rte_eal_alarm_callback cb_fn,
void *cb_arg, int rc),
rte_trace_point_emit_u64(us);
rte_trace_point_emit_ptr(cb_fn);
rte_trace_point_emit_ptr(cb_arg);
rte_trace_point_emit_int(rc);
)
RTE_TRACE_POINT(
rte_eal_trace_alarm_cancel,
RTE_TRACE_POINT_ARGS(rte_eal_alarm_callback cb_fn, void *cb_arg,
int count),
rte_trace_point_emit_ptr(cb_fn);
rte_trace_point_emit_ptr(cb_arg);
rte_trace_point_emit_int(count);
)
/* Generic */
RTE_TRACE_POINT(
rte_eal_trace_generic_void,
RTE_TRACE_POINT_ARGS(void),
)
RTE_TRACE_POINT(
rte_eal_trace_generic_u64,
RTE_TRACE_POINT_ARGS(uint64_t in),
rte_trace_point_emit_u64(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_u32,
RTE_TRACE_POINT_ARGS(uint32_t in),
rte_trace_point_emit_u32(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_u16,
RTE_TRACE_POINT_ARGS(uint16_t in),
rte_trace_point_emit_u16(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_u8,
RTE_TRACE_POINT_ARGS(uint8_t in),
rte_trace_point_emit_u8(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_i64,
RTE_TRACE_POINT_ARGS(int64_t in),
rte_trace_point_emit_i64(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_i32,
RTE_TRACE_POINT_ARGS(int32_t in),
rte_trace_point_emit_i32(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_i16,
RTE_TRACE_POINT_ARGS(int16_t in),
rte_trace_point_emit_i16(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_i8,
RTE_TRACE_POINT_ARGS(int8_t in),
rte_trace_point_emit_i8(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_int,
RTE_TRACE_POINT_ARGS(int in),
rte_trace_point_emit_int(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_long,
RTE_TRACE_POINT_ARGS(long in),
rte_trace_point_emit_long(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_float,
RTE_TRACE_POINT_ARGS(float in),
rte_trace_point_emit_float(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_double,
RTE_TRACE_POINT_ARGS(double in),
rte_trace_point_emit_double(in);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_ptr,
RTE_TRACE_POINT_ARGS(const void *ptr),
rte_trace_point_emit_ptr(ptr);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_str,
RTE_TRACE_POINT_ARGS(const char *str),
rte_trace_point_emit_string(str);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_size_t,
RTE_TRACE_POINT_ARGS(size_t sz),
rte_trace_point_emit_size_t(sz);
)
RTE_TRACE_POINT(
rte_eal_trace_generic_func,
RTE_TRACE_POINT_ARGS(const char *func),
rte_trace_point_emit_string(func);
)
#define RTE_EAL_TRACE_GENERIC_FUNC rte_eal_trace_generic_func(__func__)
/* Interrupt */
RTE_TRACE_POINT(
rte_eal_trace_intr_callback_register,
RTE_TRACE_POINT_ARGS(const struct rte_intr_handle *handle,
rte_intr_callback_fn cb, void *cb_arg, int rc),
rte_trace_point_emit_int(rc);
rte_trace_point_emit_int(handle->dev_fd);
rte_trace_point_emit_int(handle->fd);
rte_trace_point_emit_int(handle->type);
rte_trace_point_emit_u32(handle->max_intr);
rte_trace_point_emit_u32(handle->nb_efd);
rte_trace_point_emit_ptr(cb);
rte_trace_point_emit_ptr(cb_arg);
)
RTE_TRACE_POINT(
rte_eal_trace_intr_callback_unregister,
RTE_TRACE_POINT_ARGS(const struct rte_intr_handle *handle,
rte_intr_callback_fn cb, void *cb_arg, int rc),
rte_trace_point_emit_int(rc);
rte_trace_point_emit_int(handle->dev_fd);
rte_trace_point_emit_int(handle->fd);
rte_trace_point_emit_int(handle->type);
rte_trace_point_emit_u32(handle->max_intr);
rte_trace_point_emit_u32(handle->nb_efd);
rte_trace_point_emit_ptr(cb);
rte_trace_point_emit_ptr(cb_arg);
)
RTE_TRACE_POINT(
rte_eal_trace_intr_enable,
RTE_TRACE_POINT_ARGS(const struct rte_intr_handle *handle, int rc),
rte_trace_point_emit_int(rc);
rte_trace_point_emit_int(handle->dev_fd);
rte_trace_point_emit_int(handle->fd);
rte_trace_point_emit_int(handle->type);
rte_trace_point_emit_u32(handle->max_intr);
rte_trace_point_emit_u32(handle->nb_efd);
)
RTE_TRACE_POINT(
rte_eal_trace_intr_disable,
RTE_TRACE_POINT_ARGS(const struct rte_intr_handle *handle, int rc),
rte_trace_point_emit_int(rc);
rte_trace_point_emit_int(handle->dev_fd);
rte_trace_point_emit_int(handle->fd);
rte_trace_point_emit_int(handle->type);
rte_trace_point_emit_u32(handle->max_intr);
rte_trace_point_emit_u32(handle->nb_efd);
)
/* Memory */
RTE_TRACE_POINT(
rte_eal_trace_mem_zmalloc,
RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align,
int socket, void *ptr),
rte_trace_point_emit_string(type);
rte_trace_point_emit_size_t(size);
rte_trace_point_emit_u32(align);
rte_trace_point_emit_int(socket);
rte_trace_point_emit_ptr(ptr);
)
RTE_TRACE_POINT(
rte_eal_trace_mem_malloc,
RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align,
int socket, void *ptr),
rte_trace_point_emit_string(type);
rte_trace_point_emit_size_t(size);
rte_trace_point_emit_u32(align);
rte_trace_point_emit_int(socket);
rte_trace_point_emit_ptr(ptr);
)
RTE_TRACE_POINT(
rte_eal_trace_mem_realloc,
RTE_TRACE_POINT_ARGS(size_t size, unsigned int align, int socket,
void *ptr),
rte_trace_point_emit_size_t(size);
rte_trace_point_emit_u32(align);
rte_trace_point_emit_int(socket);
rte_trace_point_emit_ptr(ptr);
)
RTE_TRACE_POINT(
rte_eal_trace_mem_free,
RTE_TRACE_POINT_ARGS(void *ptr),
rte_trace_point_emit_ptr(ptr);
)
/* Memzone */
RTE_TRACE_POINT(
rte_eal_trace_memzone_reserve,
RTE_TRACE_POINT_ARGS(const char *name, size_t len, int socket_id,
unsigned int flags, unsigned int align, unsigned int bound,
const void *mz),
rte_trace_point_emit_string(name);
rte_trace_point_emit_size_t(len);
rte_trace_point_emit_int(socket_id);
rte_trace_point_emit_u32(flags);
rte_trace_point_emit_u32(align);
rte_trace_point_emit_u32(bound);
rte_trace_point_emit_ptr(mz);
)
RTE_TRACE_POINT(
rte_eal_trace_memzone_lookup,
RTE_TRACE_POINT_ARGS(const char *name, const void *memzone),
rte_trace_point_emit_string(name);
rte_trace_point_emit_ptr(memzone);
)
RTE_TRACE_POINT(
rte_eal_trace_memzone_free,
RTE_TRACE_POINT_ARGS(const char *name, void *addr, int rc),
rte_trace_point_emit_string(name);
rte_trace_point_emit_ptr(addr);
rte_trace_point_emit_int(rc);
)
/* Thread */
RTE_TRACE_POINT(
rte_eal_trace_thread_remote_launch,
RTE_TRACE_POINT_ARGS(int (*f)(void *), void *arg,
unsigned int worker_id, int rc),
rte_trace_point_emit_ptr(f);
rte_trace_point_emit_ptr(arg);
rte_trace_point_emit_u32(worker_id);
rte_trace_point_emit_int(rc);
)
RTE_TRACE_POINT(
rte_eal_trace_thread_lcore_ready,
RTE_TRACE_POINT_ARGS(unsigned int lcore_id, const char *cpuset),
rte_trace_point_emit_u32(lcore_id);
rte_trace_point_emit_string(cpuset);
)
#ifdef __cplusplus
}
#endif
#endif /* _RTE_EAL_TRACE_H_ */

116
Programs/hugePage/common/rte_epoll.h Normal file
View File

@@ -0,0 +1,116 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2021 Marvell International Ltd.
*/
#ifndef __RTE_EPOLL_H__
#define __RTE_EPOLL_H__
/**
* @file
* The rte_epoll provides interfaces functions to add delete events,
* wait poll for an event.
*/
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define RTE_INTR_EVENT_ADD 1UL
#define RTE_INTR_EVENT_DEL 2UL
typedef void (*rte_intr_event_cb_t)(int fd, void *arg);
struct rte_epoll_data {
uint32_t event; /**< event type */
void *data; /**< User data */
rte_intr_event_cb_t cb_fun; /**< IN: callback fun */
void *cb_arg; /**< IN: callback arg */
};
enum {
RTE_EPOLL_INVALID = 0,
RTE_EPOLL_VALID,
RTE_EPOLL_EXEC,
};
/** interrupt epoll event obj, taken by epoll_event.ptr */
struct rte_epoll_event {
uint32_t status; /**< OUT: event status */
int fd; /**< OUT: event fd */
int epfd; /**< OUT: epoll instance the ev associated with */
struct rte_epoll_data epdata;
};
#define RTE_EPOLL_PER_THREAD -1 /**< to hint using per thread epfd */
/**
* It waits for events on the epoll instance.
* Retries if signal received.
*
* @param epfd
* Epoll instance fd on which the caller wait for events.
* @param events
* Memory area contains the events that will be available for the caller.
* @param maxevents
* Up to maxevents are returned, must greater than zero.
* @param timeout
* Specifying a timeout of -1 causes a block indefinitely.
* Specifying a timeout equal to zero cause to return immediately.
* @return
* - On success, returns the number of available event.
* - On failure, a negative value.
*/
int
rte_epoll_wait(int epfd, struct rte_epoll_event *events,
int maxevents, int timeout);
/**
* It waits for events on the epoll instance.
* Does not retry if signal received.
*
* @param epfd
* Epoll instance fd on which the caller wait for events.
* @param events
* Memory area contains the events that will be available for the caller.
* @param maxevents
* Up to maxevents are returned, must greater than zero.
* @param timeout
* Specifying a timeout of -1 causes a block indefinitely.
* Specifying a timeout equal to zero cause to return immediately.
* @return
* - On success, returns the number of available event.
* - On failure, a negative value.
*/
int
rte_epoll_wait_interruptible(int epfd, struct rte_epoll_event *events,
int maxevents, int timeout);
/**
* It performs control operations on epoll instance referred by the epfd.
* It requests that the operation op be performed for the target fd.
*
* @param epfd
* Epoll instance fd on which the caller perform control operations.
* @param op
* The operation be performed for the target fd.
* @param fd
* The target fd on which the control ops perform.
* @param event
* Describes the object linked to the fd.
* Note: The caller must take care the object deletion after CTL_DEL.
* @return
* - On success, zero.
* - On failure, a negative value.
*/
int
rte_epoll_ctl(int epfd, int op, int fd,
struct rte_epoll_event *event);
#ifdef __cplusplus
}
#endif
#endif /* __RTE_EPOLL_H__ */

66
Programs/hugePage/common/rte_errno.h Normal file
View File

@@ -0,0 +1,66 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
/**
* @file
*
* API for error cause tracking
*/
#ifndef _RTE_ERRNO_H_
#define _RTE_ERRNO_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "rte_per_lcore.h"
RTE_DECLARE_PER_LCORE(int, _rte_errno); /**< Per core error number. */
/**
* Error number value, stored per-thread, which can be queried after
* calls to certain functions to determine why those functions failed.
*
* Uses standard values from errno.h wherever possible, with a small number
* of additional possible values for RTE-specific conditions.
*/
#define rte_errno RTE_PER_LCORE(_rte_errno)
/**
* Function which returns a printable string describing a particular
* error code. For non-RTE-specific error codes, this function returns
* the value from the libc strerror function.
*
* @param errnum
* The error number to be looked up - generally the value of rte_errno
* @return
* A pointer to a thread-local string containing the text describing
* the error.
*/
const char *rte_strerror(int errnum);
#ifndef __ELASTERROR
/**
* Check if we have a defined value for the max system-defined errno values.
* if no max defined, start from 1000 to prevent overlap with standard values
*/
#define __ELASTERROR 1000
#endif
/** Error types */
enum {
RTE_MIN_ERRNO = __ELASTERROR, /**< Start numbering above std errno vals */
E_RTE_SECONDARY, /**< Operation not allowed in secondary processes */
E_RTE_NO_CONFIG, /**< Missing rte_config */
RTE_MAX_ERRNO /**< Max RTE error number */
};
#ifdef __cplusplus
}
#endif
#endif /* _RTE_ERRNO_H_ */

538
Programs/hugePage/common/rte_fbarray.h Normal file
View File

@@ -0,0 +1,538 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017-2018 Intel Corporation
*/
#ifndef RTE_FBARRAY_H
#define RTE_FBARRAY_H
/**
* @file
*
* File-backed shared indexed array for DPDK.
*
* Basic workflow is expected to be the following:
* 1) Allocate array either using ``rte_fbarray_init()`` or
* ``rte_fbarray_attach()`` (depending on whether it's shared between
* multiple DPDK processes)
* 2) find free spots using ``rte_fbarray_find_next_free()``
* 3) get pointer to data in the free spot using ``rte_fbarray_get()``, and
* copy data into the pointer (element size is fixed)
* 4) mark entry as used using ``rte_fbarray_set_used()``
*
* Calls to ``rte_fbarray_init()`` and ``rte_fbarray_destroy()`` will have
* consequences for all processes, while calls to ``rte_fbarray_attach()`` and
* ``rte_fbarray_detach()`` will only have consequences within a single process.
* Therefore, it is safe to call ``rte_fbarray_attach()`` or
* ``rte_fbarray_detach()`` while another process is using ``rte_fbarray``,
* provided no other thread within the same process will try to use
* ``rte_fbarray`` before attaching or after detaching. It is not safe to call
* ``rte_fbarray_init()`` or ``rte_fbarray_destroy()`` while another thread or
* another process is using ``rte_fbarray``.
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include "generic/rte_rwlock.h"
#define RTE_FBARRAY_NAME_LEN 64
struct rte_fbarray {
char name[RTE_FBARRAY_NAME_LEN]; /**< name associated with an array */
unsigned int count; /**< number of entries stored */
unsigned int len; /**< current length of the array */
unsigned int elt_sz; /**< size of each element */
void *data; /**< data pointer */
rte_rwlock_t rwlock; /**< multiprocess lock */
};
/**
* Set up ``rte_fbarray`` structure and allocate underlying resources.
*
* Call this function to correctly set up ``rte_fbarray`` and allocate
* underlying files that will be backing the data in the current process. Note
* that in order to use and share ``rte_fbarray`` between multiple processes,
* data pointed to by ``arr`` pointer must itself be allocated in shared memory.
*
* @param arr
* Valid pointer to allocated ``rte_fbarray`` structure.
*
* @param name
* Unique name to be assigned to this array.
*
* @param len
* Number of elements initially available in the array.
*
* @param elt_sz
* Size of each element.
*
* @return
* - 0 on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,
unsigned int elt_sz);
/**
* Attach to a file backing an already allocated and correctly set up
* ``rte_fbarray`` structure.
*
* Call this function to attach to file that will be backing the data in the
* current process. The structure must have been previously correctly set up
* with a call to ``rte_fbarray_init()``. Calls to ``rte_fbarray_attach()`` are
* usually meant to be performed in a multiprocessing scenario, with data
* pointed to by ``arr`` pointer allocated in shared memory.
*
* @param arr
* Valid pointer to allocated and correctly set up rte_fbarray structure.
*
* @return
* - 0 on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_attach(struct rte_fbarray *arr);
/**
* Deallocate resources for an already allocated and correctly set up
* ``rte_fbarray`` structure, and remove the underlying file.
*
* Call this function to deallocate all resources associated with an
* ``rte_fbarray`` structure within the current process. This will also
* zero-fill data pointed to by ``arr`` pointer and remove the underlying file
* backing the data, so it is expected that by the time this function is called,
* all other processes have detached from this ``rte_fbarray``.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @return
* - 0 on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_destroy(struct rte_fbarray *arr);
/**
* Deallocate resources for an already allocated and correctly set up
* ``rte_fbarray`` structure.
*
* Call this function to deallocate all resources associated with an
* ``rte_fbarray`` structure within current process.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @return
* - 0 on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_detach(struct rte_fbarray *arr);
/**
* Get pointer to element residing at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param idx
* Index of an element to get a pointer to.
*
* @return
* - non-NULL pointer on success.
* - NULL on failure, with ``rte_errno`` indicating reason for failure.
*/
void *
rte_fbarray_get(const struct rte_fbarray *arr, unsigned int idx);
/**
* Find index of a specified element within the array.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param elt
* Pointer to element to find index to.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_idx(const struct rte_fbarray *arr, const void *elt);
/**
* Mark specified element as used.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param idx
* Element index to mark as used.
*
* @return
* - 0 on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_set_used(struct rte_fbarray *arr, unsigned int idx);
/**
* Mark specified element as free.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param idx
* Element index to mark as free.
*
* @return
* - 0 on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_set_free(struct rte_fbarray *arr, unsigned int idx);
/**
* Check whether element at specified index is marked as used.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param idx
* Element index to check as used.
*
* @return
* - 1 if element is used.
* - 0 if element is unused.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_is_used(struct rte_fbarray *arr, unsigned int idx);
/**
* Find index of next free element, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of next used element, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_next_used(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of next chunk of ``n`` free elements, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @param n
* Number of free elements to look for.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_next_n_free(struct rte_fbarray *arr, unsigned int start,
unsigned int n);
/**
* Find index of next chunk of ``n`` used elements, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @param n
* Number of used elements to look for.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_next_n_used(struct rte_fbarray *arr, unsigned int start,
unsigned int n);
/**
* Find how many more free entries there are, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_contig_free(struct rte_fbarray *arr,
unsigned int start);
/**
* Find how many more used entries there are, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of previous free element, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_prev_free(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of previous used element, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_prev_used(struct rte_fbarray *arr, unsigned int start);
/**
* Find lowest start index of chunk of ``n`` free elements, down from specified
* index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @param n
* Number of free elements to look for.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_prev_n_free(struct rte_fbarray *arr, unsigned int start,
unsigned int n);
/**
* Find lowest start index of chunk of ``n`` used elements, down from specified
* index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @param n
* Number of used elements to look for.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_prev_n_used(struct rte_fbarray *arr, unsigned int start,
unsigned int n);
/**
* Find how many more free entries there are before specified index (like
* ``rte_fbarray_find_contig_free`` but going in reverse).
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_rev_contig_free(struct rte_fbarray *arr,
unsigned int start);
/**
* Find how many more used entries there are before specified index (like
* ``rte_fbarray_find_contig_used`` but going in reverse).
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_rev_contig_used(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of biggest chunk of free elements, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_biggest_free(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of biggest chunk of used elements, starting at specified index.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_biggest_used(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of biggest chunk of free elements before a specified index (like
* ``rte_fbarray_find_biggest_free``, but going in reverse).
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_rev_biggest_free(struct rte_fbarray *arr, unsigned int start);
/**
* Find index of biggest chunk of used elements before a specified index (like
* ``rte_fbarray_find_biggest_used``, but going in reverse).
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param start
* Element index to start search from.
*
* @return
* - non-negative integer on success.
* - -1 on failure, with ``rte_errno`` indicating reason for failure.
*/
int
rte_fbarray_find_rev_biggest_used(struct rte_fbarray *arr, unsigned int start);
/**
* Dump ``rte_fbarray`` metadata.
*
* @param arr
* Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
*
* @param f
* File object to dump information into.
*/
void
rte_fbarray_dump_metadata(struct rte_fbarray *arr, FILE *f);
#ifdef __cplusplus
}
#endif
#endif /* RTE_FBARRAY_H */

99
Programs/hugePage/common/rte_function_versioning.h Normal file
View File

@@ -0,0 +1,99 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2015 Neil Horman <nhorman@tuxdriver.com>.
* All rights reserved.
*/
#ifndef _RTE_FUNCTION_VERSIONING_H_
#define _RTE_FUNCTION_VERSIONING_H_
#include "rte_common.h"
#ifndef RTE_USE_FUNCTION_VERSIONING
#error Use of function versioning disabled, is "use_function_versioning=true" in meson.build?
#endif
#ifdef RTE_BUILD_SHARED_LIB
/*
* Provides backwards compatibility when updating exported functions.
* When a symbol is exported from a library to provide an API, it also provides a
* calling convention (ABI) that is embodied in its name, return type,
* arguments, etc. On occasion that function may need to change to accommodate
* new functionality, behavior, etc. When that occurs, it is desirable to
* allow for backwards compatibility for a time with older binaries that are
* dynamically linked to the dpdk. To support that, the __vsym and
* VERSION_SYMBOL macros are created. They, in conjunction with the
* version.map file for a given library allow for multiple versions of
* a symbol to exist in a shared library so that older binaries need not be
* immediately recompiled.
*
* Refer to the guidelines document in the docs subdirectory for details on the
* use of these macros
*/
/*
* Macro Parameters:
* b - function base name
* e - function version extension, to be concatenated with base name
* n - function symbol version string to be applied
* f - function prototype
* p - full function symbol name
*/
/*
* VERSION_SYMBOL
* Creates a symbol version table entry binding symbol <b>@DPDK_<n> to the internal
* function name <b><e>
*/
#define VERSION_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@DPDK_" RTE_STR(n))
/*
* VERSION_SYMBOL_EXPERIMENTAL
* Creates a symbol version table entry binding the symbol <b>@EXPERIMENTAL to the internal
* function name <b><e>. The macro is used when a symbol matures to become part of the stable ABI,
* to provide an alias to experimental for some time.
*/
#define VERSION_SYMBOL_EXPERIMENTAL(b, e) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@EXPERIMENTAL")
/*
* BIND_DEFAULT_SYMBOL
* Creates a symbol version entry instructing the linker to bind references to
* symbol <b> to the internal symbol <b><e>
*/
#define BIND_DEFAULT_SYMBOL(b, e, n) __asm__(".symver " RTE_STR(b) RTE_STR(e) ", " RTE_STR(b) "@@DPDK_" RTE_STR(n))
/*
* __vsym
* Annotation to be used in declaration of the internal symbol <b><e> to signal
* that it is being used as an implementation of a particular version of symbol
* <b>.
*/
#define __vsym __rte_used
/*
* MAP_STATIC_SYMBOL
* If a function has been bifurcated into multiple versions, none of which
* are defined as the exported symbol name in the map file, this macro can be
* used to alias a specific version of the symbol to its exported name. For
* example, if you have 2 versions of a function foo_v1 and foo_v2, where the
* former is mapped to foo@DPDK_1 and the latter is mapped to foo@DPDK_2 when
* building a shared library, this macro can be used to map either foo_v1 or
* foo_v2 to the symbol foo when building a static library, e.g.:
* MAP_STATIC_SYMBOL(void foo(), foo_v2);
*/
#define MAP_STATIC_SYMBOL(f, p)
#else
/*
* No symbol versioning in use
*/
#define VERSION_SYMBOL(b, e, n)
#define VERSION_SYMBOL_EXPERIMENTAL(b, e)
#define __vsym
#define BIND_DEFAULT_SYMBOL(b, e, n)
#define MAP_STATIC_SYMBOL(f, p) f __attribute__((alias(RTE_STR(p))))
/*
* RTE_BUILD_SHARED_LIB=n
*/
#endif
#endif /* _RTE_FUNCTION_VERSIONING_H_ */

Some files were not shown because too many files have changed in this diff Show More