xref: /gem5/ext/mcpat/cacti/nuca.cc

/*****************************************************************************
 *                                McPAT/CACTI
 *                      SOFTWARE LICENSE AGREEMENT
 *            Copyright 2012 Hewlett-Packard Development Company, L.P.
 *            Copyright (c) 2010-2013 Advanced Micro Devices, Inc.
 *                          All Rights Reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.

 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 ***************************************************************************/



#include <cassert>

#include "Ucache.h"
#include "nuca.h"

unsigned int MIN_BANKSIZE = 65536;
#define FIXED_OVERHEAD 55e-12 /* clock skew and jitter in s. Ref: Hrishikesh et al ISCA 01 */
#define LATCH_DELAY 28e-12 /* latch delay in s (later should use FO4 TODO) */
#define CONTR_2_BANK_LAT 0

int cont_stats[2 /*l2 or l3*/][5/* cores */][ROUTER_TYPES][7 /*banks*/][8 /* cycle time */];

Nuca::Nuca(
    TechnologyParameter::DeviceType *dt = &(g_tp.peri_global)
): deviceType(dt) {
    init_cont();
}

void
Nuca::init_cont() {
    FILE *cont;
    char line[5000];
    char jk[5000];
    cont = fopen("contention.dat", "r");
    if (!cont) {
        cout << "contention.dat file is missing!\n";
        exit(0);
    }

    for (int i = 0; i < 2; i++) {
        for (int j = 2; j < 5; j++) {
            for (int k = 0; k < ROUTER_TYPES; k++) {
                for (int l = 0; l < 7; l++) {
                    int *temp = cont_stats[i/*l2 or l3*/][j/*core*/][k/*64 or 128 or 256 link bw*/][l /* no banks*/];
                    assert(fscanf(cont, "%[^\n]\n", line) != EOF);
                    sscanf(line, "%[^:]: %d %d %d %d %d %d %d %d", jk,
                           &temp[0], &temp[1], &temp[2], &temp[3],
                           &temp[4], &temp[5], &temp[6], &temp[7]);
                }
            }
        }
    }
    fclose(cont);
}

void
Nuca::print_cont_stats() {
    for (int i = 0; i < 2; i++) {
        for (int j = 2; j < 5; j++) {
            for (int k = 0; k < ROUTER_TYPES; k++) {
                for (int l = 0; l < 7; l++) {
                    for (int m = 0; l < 7; l++) {
                        cout << cont_stats[i][j][k][l][m] << " ";
                    }
                    cout << endl;
                }
            }
        }
    }
    cout << endl;
}

Nuca::~Nuca() {
    for (int i = wt_min; i <= wt_max; i++) {
        delete wire_vertical[i];
        delete wire_horizontal[i];
    }
}

/* converts latency (in s) to cycles depending upon the FREQUENCY (in GHz) */
int
Nuca::calc_cycles(double lat, double oper_freq) {
    //TODO: convert latch delay to FO4 */
    double cycle_time = (1.0 / (oper_freq * 1e9)); /*s*/
    cycle_time -= LATCH_DELAY;
    cycle_time -= FIXED_OVERHEAD;

    return (int)ceil(lat / cycle_time);
}


nuca_org_t::~nuca_org_t() {
    // if(h_wire) delete h_wire;
    // if(v_wire) delete v_wire;
    // if(router) delete router;
}

/*
 * Version - 6.0
 *
 * Perform exhaustive search across different bank organizatons,
 * router configurations, grid organizations, and wire models and
 * find an optimal NUCA organization
 * For different bank count values
 * 1. Optimal bank organization is calculated
 * 2. For each bank organization, find different NUCA organizations
 *    using various router configurations, grid organizations,
 *    and wire models.
 * 3. NUCA model with the least cost is picked for
 *    this particular bank count
 * Finally include contention statistics and find the optimal
 *    NUCA configuration
 */
void
Nuca::sim_nuca() {
    /* temp variables */
    int it, ro, wr;
    int num_cyc;
    unsigned int i, j, k;
    unsigned int r, c;
    int l2_c;
    int bank_count = 0;
    uca_org_t ures;
    nuca_org_t *opt_n;
    mem_array tag, data;
    list<nuca_org_t *> nuca_list;
    Router *router_s[ROUTER_TYPES];
    router_s[0] = new Router(64.0, 8, 4, &(g_tp.peri_global));
    router_s[0]->print_router();
    router_s[1] = new Router(128.0, 8, 4, &(g_tp.peri_global));
    router_s[1]->print_router();
    router_s[2] = new Router(256.0, 8, 4, &(g_tp.peri_global));
    router_s[2]->print_router();

    int core_in; // to store no. of cores

    /* to search diff grid organizations */
    double curr_hop, totno_hops, totno_hhops, totno_vhops, tot_lat,
    curr_acclat;
    double avg_lat, avg_hop, avg_hhop, avg_vhop, avg_dyn_power,
    avg_leakage_power;

    double opt_acclat = INF, opt_avg_lat = INF, opt_tot_lat = INF;
    int opt_rows = 0;
    int opt_columns = 0;
    double opt_totno_hops = 0;
    double opt_avg_hop = 0;
    double opt_dyn_power = 0, opt_leakage_power = 0;
    min_values_t minval;

    int bank_start = 0;

    int flit_width = 0;

    /* vertical and horizontal hop latency values */
    int ver_hop_lat, hor_hop_lat; /* in cycles */


    /* no. of different bank sizes to consider */
    int iterations;


    g_ip->nuca_cache_sz = g_ip->cache_sz;
    nuca_list.push_back(new nuca_org_t());

    if (g_ip->cache_level == 0) l2_c = 1;
    else l2_c = 0;

    if (g_ip->cores <= 4) core_in = 2;
    else if (g_ip->cores <= 8) core_in = 3;
    else if (g_ip->cores <= 16) core_in = 4;
    else {
        cout << "Number of cores should be <= 16!\n";
        exit(0);
    }


    // set the lower bound to an appropriate value. this depends on cache associativity
    if (g_ip->assoc > 2) {
        i = 2;
        while (i != g_ip->assoc) {
            MIN_BANKSIZE *= 2;
            i *= 2;
        }
    }

    iterations = (int)logtwo((int)g_ip->cache_sz / MIN_BANKSIZE);

    if (g_ip->force_wiretype) {
        if (g_ip->wt == Low_swing) {
            wt_min = Low_swing;
            wt_max = Low_swing;
        } else {
            wt_min = Global;
            wt_max = Low_swing - 1;
        }
    } else {
        wt_min = Global;
        wt_max = Low_swing;
    }
    if (g_ip->nuca_bank_count != 0) { // simulate just one bank
        if (g_ip->nuca_bank_count != 2 && g_ip->nuca_bank_count != 4 &&
                g_ip->nuca_bank_count != 8 && g_ip->nuca_bank_count != 16 &&
                g_ip->nuca_bank_count != 32 && g_ip->nuca_bank_count != 64) {
            fprintf(stderr, "Incorrect bank count value! Please fix the ",
                    "value in cache.cfg\n");
        }
        bank_start = (int)logtwo((double)g_ip->nuca_bank_count);
        iterations = bank_start + 1;
        g_ip->cache_sz = g_ip->cache_sz / g_ip->nuca_bank_count;
    }
    cout << "Simulating various NUCA configurations\n";
    for (it = bank_start; it < iterations; it++) {
        /* different bank count values */
        ures.tag_array2 = &tag;
        ures.data_array2 = &data;
        /*
         * find the optimal bank organization
         */
        solve(&ures);
//    output_UCA(&ures);
        bank_count = g_ip->nuca_cache_sz / g_ip->cache_sz;
        cout << "====" <<  g_ip->cache_sz << "\n";

        for (wr = wt_min; wr <= wt_max; wr++) {

            for (ro = 0; ro < ROUTER_TYPES; ro++) {
                flit_width = (int) router_s[ro]->flit_size; //initialize router
                nuca_list.back()->nuca_pda.cycle_time = router_s[ro]->cycle_time;

                /* calculate router and wire parameters */

                double vlength = ures.cache_ht; /* length of the wire (u)*/
                double hlength = ures.cache_len; // u

                /* find delay, area, and power for wires */
                wire_vertical[wr] = new Wire((enum Wire_type) wr, vlength);
                wire_horizontal[wr] = new Wire((enum Wire_type) wr, hlength);


                hor_hop_lat =
                    calc_cycles(wire_horizontal[wr]->delay,
                                1 /(nuca_list.back()->nuca_pda.cycle_time *
                                    .001));
                ver_hop_lat =
                    calc_cycles(wire_vertical[wr]->delay,
                                1 / (nuca_list.back()->nuca_pda.cycle_time *
                                     .001));

                /*
                 * assume a grid like topology and explore for optimal network
                 * configuration using different row and column count values.
                 */
                for (c = 1; c <= (unsigned int)bank_count; c++) {
                    while (bank_count % c != 0) c++;
                    r = bank_count / c;

                    /*
                     * to find the avg access latency of a NUCA cache, uncontended
                     * access time to each bank from the
                     * cache controller is calculated.
                     * avg latency =
                     * sum of the access latencies to individual banks)/bank
                     * count value.
                     */
                    totno_hops = totno_hhops = totno_vhops = tot_lat = 0;
                    k = 1;
                    for (i = 0; i < r; i++) {
                        for (j = 0; j < c; j++) {
                            /*
                             * vertical hops including the
                             * first hop from the cache controller
                             */
                            curr_hop = i + 1;
                            curr_hop += j; /* horizontal hops */
                            totno_hhops += j;
                            totno_vhops += (i + 1);
                            curr_acclat = (i * ver_hop_lat + CONTR_2_BANK_LAT +
                                           j * hor_hop_lat);

                            tot_lat += curr_acclat;
                            totno_hops += curr_hop;
                        }
                    }
                    avg_lat = tot_lat / bank_count;
                    avg_hop = totno_hops / bank_count;
                    avg_hhop = totno_hhops / bank_count;
                    avg_vhop = totno_vhops / bank_count;

                    /* net access latency */
                    curr_acclat = 2 * avg_lat + 2 * (router_s[ro]->delay *
                                                     avg_hop) +
                        calc_cycles(ures.access_time,
                                    1 /
                                    (nuca_list.back()->nuca_pda.cycle_time *
                                     .001));

                    /* avg access lat of nuca */
                    avg_dyn_power =
                        avg_hop *
                        (router_s[ro]->power.readOp.dynamic) + avg_hhop *
                        (wire_horizontal[wr]->power.readOp.dynamic) *
                        (g_ip->block_sz * 8 + 64) + avg_vhop *
                        (wire_vertical[wr]->power.readOp.dynamic) *
                        (g_ip->block_sz * 8 + 64) + ures.power.readOp.dynamic;

                    avg_leakage_power =
                        bank_count * router_s[ro]->power.readOp.leakage +
                        avg_hhop * (wire_horizontal[wr]->power.readOp.leakage *
                                    wire_horizontal[wr]->delay) * flit_width +
                        avg_vhop * (wire_vertical[wr]->power.readOp.leakage *
                                    wire_horizontal[wr]->delay);

                    if (curr_acclat < opt_acclat) {
                        opt_acclat = curr_acclat;
                        opt_tot_lat = tot_lat;
                        opt_avg_lat = avg_lat;
                        opt_totno_hops = totno_hops;
                        opt_avg_hop = avg_hop;
                        opt_rows = r;
                        opt_columns = c;
                        opt_dyn_power = avg_dyn_power;
                        opt_leakage_power = avg_leakage_power;
                    }
                    totno_hops = 0;
                    tot_lat = 0;
                    totno_hhops = 0;
                    totno_vhops = 0;
                }
                nuca_list.back()->wire_pda.power.readOp.dynamic =
                    opt_avg_hop * flit_width *
                    (wire_horizontal[wr]->power.readOp.dynamic +
                     wire_vertical[wr]->power.readOp.dynamic);
                nuca_list.back()->avg_hops = opt_avg_hop;
                /* network delay/power */
                nuca_list.back()->h_wire = wire_horizontal[wr];
                nuca_list.back()->v_wire = wire_vertical[wr];
                nuca_list.back()->router = router_s[ro];
                /* bank delay/power */

                nuca_list.back()->bank_pda.delay = ures.access_time;
                nuca_list.back()->bank_pda.power = ures.power;
                nuca_list.back()->bank_pda.area.h = ures.cache_ht;
                nuca_list.back()->bank_pda.area.w = ures.cache_len;
                nuca_list.back()->bank_pda.cycle_time = ures.cycle_time;

                num_cyc = calc_cycles(nuca_list.back()->bank_pda.delay /*s*/,
                                      1 /
                                      (nuca_list.back()->nuca_pda.cycle_time *
                                       .001/*GHz*/));
                if (num_cyc % 2 != 0) num_cyc++;
                if (num_cyc > 16) num_cyc = 16; // we have data only up to 16 cycles

                if (it < 7) {
                    nuca_list.back()->nuca_pda.delay = opt_acclat +
                                                       cont_stats[l2_c][core_in][ro][it][num_cyc/2-1];
                    nuca_list.back()->contention =
                        cont_stats[l2_c][core_in][ro][it][num_cyc/2-1];
                } else {
                    nuca_list.back()->nuca_pda.delay = opt_acclat +
                                                       cont_stats[l2_c][core_in][ro][7][num_cyc/2-1];
                    nuca_list.back()->contention =
                        cont_stats[l2_c][core_in][ro][7][num_cyc/2-1];
                }
                nuca_list.back()->nuca_pda.power.readOp.dynamic = opt_dyn_power;
                nuca_list.back()->nuca_pda.power.readOp.leakage = opt_leakage_power;

                /* array organization */
                nuca_list.back()->bank_count = bank_count;
                nuca_list.back()->rows = opt_rows;
                nuca_list.back()->columns = opt_columns;
                calculate_nuca_area (nuca_list.back());

                minval.update_min_values(nuca_list.back());
                nuca_list.push_back(new nuca_org_t());
                opt_acclat = BIGNUM;

            }
        }
        g_ip->cache_sz /= 2;
    }

    delete(nuca_list.back());
    nuca_list.pop_back();
    opt_n = find_optimal_nuca(&nuca_list, &minval);
    print_nuca(opt_n);
    g_ip->cache_sz = g_ip->nuca_cache_sz / opt_n->bank_count;

    list<nuca_org_t *>::iterator niter;
    for (niter = nuca_list.begin(); niter != nuca_list.end(); ++niter) {
        delete *niter;
    }
    nuca_list.clear();

    for (int i = 0; i < ROUTER_TYPES; i++) {
        delete router_s[i];
    }
    g_ip->display_ip();
    //  g_ip->force_cache_config = true;
    //  g_ip->ndwl = 8;
    //  g_ip->ndbl = 16;
    //  g_ip->nspd = 4;
    //  g_ip->ndcm = 1;
    //  g_ip->ndsam1 = 8;
    //  g_ip->ndsam2 = 32;

}


void
Nuca::print_nuca (nuca_org_t *fr) {
    printf("\n---------- CACTI version 6.5, Non-uniform Cache Access "
           "----------\n\n");
    printf("Optimal number of banks - %d\n", fr->bank_count);
    printf("Grid organization rows x columns - %d x %d\n",
           fr->rows, fr->columns);
    printf("Network frequency - %g GHz\n",
           (1 / fr->nuca_pda.cycle_time)*1e3);
    printf("Cache dimension (mm x mm) - %g x %g\n",
           fr->nuca_pda.area.h,
           fr->nuca_pda.area.w);

    fr->router->print_router();

    printf("\n\nWire stats:\n");
    if (fr->h_wire->wt == Global) {
        printf("\tWire type - Full swing global wires with least "
               "possible delay\n");
    } else if (fr->h_wire->wt == Global_5) {
        printf("\tWire type - Full swing global wires with "
               "5%% delay penalty\n");
    } else if (fr->h_wire->wt == Global_10) {
        printf("\tWire type - Full swing global wires with "
               "10%% delay penalty\n");
    } else if (fr->h_wire->wt == Global_20) {
        printf("\tWire type - Full swing global wires with "
               "20%% delay penalty\n");
    } else if (fr->h_wire->wt == Global_30) {
        printf("\tWire type - Full swing global wires with "
               "30%% delay penalty\n");
    } else if (fr->h_wire->wt == Low_swing) {
        printf("\tWire type - Low swing wires\n");
    }

    printf("\tHorizontal link delay - %g (ns)\n",
           fr->h_wire->delay*1e9);
    printf("\tVertical link delay - %g (ns)\n",
           fr->v_wire->delay*1e9);
    printf("\tDelay/length - %g (ns/mm)\n",
           fr->h_wire->delay*1e9 / fr->bank_pda.area.w);
    printf("\tHorizontal link energy -dynamic/access %g (nJ)\n"
           "\t                       -leakage %g (nW)\n\n",
           fr->h_wire->power.readOp.dynamic*1e9,
           fr->h_wire->power.readOp.leakage*1e9);
    printf("\tVertical link energy -dynamic/access %g (nJ)\n"
           "\t                     -leakage %g (nW)\n\n",
           fr->v_wire->power.readOp.dynamic*1e9,
           fr->v_wire->power.readOp.leakage*1e9);
    printf("\n\n");
    fr->v_wire->print_wire();
    printf("\n\nBank stats:\n");
}


nuca_org_t *
Nuca::find_optimal_nuca (list<nuca_org_t *> *n, min_values_t *minval) {
    double cost = 0;
    double min_cost = BIGNUM;
    nuca_org_t *res = NULL;
    float d, a, dp, lp, c;
    int v;
    dp = g_ip->dynamic_power_wt_nuca;
    lp = g_ip->leakage_power_wt_nuca;
    a = g_ip->area_wt_nuca;
    d = g_ip->delay_wt_nuca;
    c = g_ip->cycle_time_wt_nuca;

    list<nuca_org_t *>::iterator niter;


    for (niter = n->begin(); niter != n->end(); niter++) {
        fprintf(stderr, "\n-----------------------------"
                "---------------\n");


        printf("NUCA___stats %d \tbankcount: lat = %g \tdynP = %g \twt = %d\t "
               "bank_dpower = %g \tleak = %g \tcycle = %g\n",
               (*niter)->bank_count,
               (*niter)->nuca_pda.delay,
               (*niter)->nuca_pda.power.readOp.dynamic,
               (*niter)->h_wire->wt,
               (*niter)->bank_pda.power.readOp.dynamic,
               (*niter)->nuca_pda.power.readOp.leakage,
               (*niter)->nuca_pda.cycle_time);


        if (g_ip->ed == 1) {
            cost = ((*niter)->nuca_pda.delay / minval->min_delay) *
                   ((*niter)->nuca_pda.power.readOp.dynamic / minval->min_dyn);
            if (min_cost > cost) {
                min_cost = cost;
                res = ((*niter));
            }
        } else if (g_ip->ed == 2) {
            cost = ((*niter)->nuca_pda.delay / minval->min_delay) *
                   ((*niter)->nuca_pda.delay / minval->min_delay) *
                   ((*niter)->nuca_pda.power.readOp.dynamic / minval->min_dyn);
            if (min_cost > cost) {
                min_cost = cost;
                res = ((*niter));
            }
        } else {
            /*
             * check whether the current organization
             * meets the input deviation constraints
             */
            v = check_nuca_org((*niter), minval);
            if (minval->min_leakage == 0) minval->min_leakage = 0.1; //FIXME remove this after leakage modeling

            if (v) {
                cost = (d  * ((*niter)->nuca_pda.delay / minval->min_delay) +
                        c  * ((*niter)->nuca_pda.cycle_time / minval->min_cyc) +
                        dp * ((*niter)->nuca_pda.power.readOp.dynamic /
                              minval->min_dyn) +
                        lp * ((*niter)->nuca_pda.power.readOp.leakage /
                              minval->min_leakage) +
                        a  * ((*niter)->nuca_pda.area.get_area() /
                              minval->min_area));
                fprintf(stderr, "cost = %g\n", cost);

                if (min_cost > cost) {
                    min_cost = cost;
                    res = ((*niter));
                }
            } else {
                niter = n->erase(niter);
                if (niter != n->begin())
                    niter --;
            }
        }
    }
    return res;
}

int
Nuca::check_nuca_org (nuca_org_t *n, min_values_t *minval) {
    if (((n->nuca_pda.delay - minval->min_delay)*100 / minval->min_delay) >
        g_ip->delay_dev_nuca) {
        return 0;
    }
    if (((n->nuca_pda.power.readOp.dynamic - minval->min_dyn) /
         minval->min_dyn)*100 >
        g_ip->dynamic_power_dev_nuca) {
        return 0;
    }
    if (((n->nuca_pda.power.readOp.leakage - minval->min_leakage) /
         minval->min_leakage)*100 >
        g_ip->leakage_power_dev_nuca) {
        return 0;
    }
    if (((n->nuca_pda.cycle_time - minval->min_cyc) / minval->min_cyc)*100 >
        g_ip->cycle_time_dev_nuca) {
        return 0;
    }
    if (((n->nuca_pda.area.get_area() - minval->min_area) / minval->min_area) *
        100 >
        g_ip->area_dev_nuca) {
        return 0;
    }
    return 1;
}

void
Nuca::calculate_nuca_area (nuca_org_t *nuca) {
    nuca->nuca_pda.area.h =
        nuca->rows * ((nuca->h_wire->wire_width +
                       nuca->h_wire->wire_spacing)
                      * nuca->router->flit_size +
                      nuca->bank_pda.area.h);

    nuca->nuca_pda.area.w =
        nuca->columns * ((nuca->v_wire->wire_width +
                          nuca->v_wire->wire_spacing)
                         * nuca->router->flit_size +
                         nuca->bank_pda.area.w);
}